From 306feb87f055f7c2a3c9b782a84bdc613656b922 Mon Sep 17 00:00:00 2001 From: Georg Traar Date: Wed, 26 Jun 2024 23:01:35 +0200 Subject: [PATCH 1/3] Framework/Streamlit: Add basic example about Streamlit, reading `sys.summits` --- framework/streamlit/.streamlit/secrets.toml | 8 +++++ framework/streamlit/README.md | 34 +++++++++++++++++++++ framework/streamlit/basic_sys_summits.py | 25 +++++++++++++++ framework/streamlit/requirements.txt | 2 ++ 4 files changed, 69 insertions(+) create mode 100644 framework/streamlit/.streamlit/secrets.toml create mode 100644 framework/streamlit/README.md create mode 100644 framework/streamlit/basic_sys_summits.py create mode 100644 framework/streamlit/requirements.txt diff --git a/framework/streamlit/.streamlit/secrets.toml b/framework/streamlit/.streamlit/secrets.toml new file mode 100644 index 00000000..f939975d --- /dev/null +++ b/framework/streamlit/.streamlit/secrets.toml @@ -0,0 +1,8 @@ +# .streamlit/secrets.toml + +[connections.cratedb] +dialect = "crate" +host = "localhost" +port = "4200" +username = "crate" +password = "" diff --git a/framework/streamlit/README.md b/framework/streamlit/README.md new file mode 100644 index 00000000..665ff814 --- /dev/null +++ b/framework/streamlit/README.md @@ -0,0 +1,34 @@ +# Streamlit with CrateDB Example + +## About +Demonstrate connectivity from Streamlit to CrateDB. + +## Configuration +Configure database connection address and credentials within +`.streamlit/secrets.toml` in your working directory. Please +make sure to use valid credentials matching your environment. + +## Usage +To start a CrateDB instance on your machine, invoke: +```shell +docker run -it --rm \ + --publish=4200:4200 --publish=5432:5432 \ + --env=CRATE_HEAP_SIZE=2g \ + crate:latest -Cdiscovery.type=single-node +``` + +Install dependencies. +```shell +pip install -r requirements.txt +``` + +Invoke Streamlit to serve the application. +```shell +streamlit run basic_sys_summits.py +``` + +## Screenshot + +Enjoy the list of mountains. + +![image](https://github.com/crate/cratedb-examples/assets/453543/7dc54224-06d0-4cfb-a5e0-b216c03bf3d2) diff --git a/framework/streamlit/basic_sys_summits.py b/framework/streamlit/basic_sys_summits.py new file mode 100644 index 00000000..bf67ec86 --- /dev/null +++ b/framework/streamlit/basic_sys_summits.py @@ -0,0 +1,25 @@ +""" +A basic Streamlit application connecting to CrateDB using SQLAlchemy. + +It reads the built-in `sys.summits` table into a dataframe, and +displays its contents. + +- https://docs.streamlit.io/develop/tutorials/databases +- https://cratedb.com/docs/sqlalchemy-cratedb/ +""" +import streamlit as st + +# Set a title for the page. +st.title("Streamlit with CrateDB Example") + +# Connect to CrateDB, and read the built-in `sys.summits` table. +conn = st.connection("cratedb", type="sql") +df = conn.query('SELECT * FROM "sys"."summits";', ttl="10m") + +# Output data as dataframe and table. +st.dataframe(df) +st.table(df) + +# Output data as Markdown. +for row in df.itertuples(): + st.write(f"{row.mountain}") diff --git a/framework/streamlit/requirements.txt b/framework/streamlit/requirements.txt new file mode 100644 index 00000000..ef1e6c40 --- /dev/null +++ b/framework/streamlit/requirements.txt @@ -0,0 +1,2 @@ +streamlit==1.* +sqlalchemy-cratedb==0.38.0 From 40d357041e506382495c5aad2be7de7f72fe8221 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 26 Jun 2024 23:02:32 +0200 Subject: [PATCH 2/3] Framework/Streamlit: Add software tests --- .github/dependabot.yml | 5 ++ .github/workflows/framework-streamlit.yml | 74 +++++++++++++++++++++++ framework/streamlit/README.md | 17 ++++++ framework/streamlit/pyproject.toml | 12 ++++ framework/streamlit/requirements-dev.txt | 1 + framework/streamlit/test.py | 29 +++++++++ 6 files changed, 138 insertions(+) create mode 100644 .github/workflows/framework-streamlit.yml create mode 100644 framework/streamlit/pyproject.toml create mode 100644 framework/streamlit/requirements-dev.txt create mode 100644 framework/streamlit/test.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d2e05394..55c32ca6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -80,6 +80,11 @@ updates: schedule: interval: "daily" + - directory: "/framework/streamlit" + package-ecosystem: "pip" + schedule: + interval: "daily" + # Topics. - directory: "/topic/machine-learning/automl" diff --git a/.github/workflows/framework-streamlit.yml b/.github/workflows/framework-streamlit.yml new file mode 100644 index 00000000..7e82e394 --- /dev/null +++ b/.github/workflows/framework-streamlit.yml @@ -0,0 +1,74 @@ +name: Streamlit + +on: + pull_request: + branches: ~ + paths: + - '.github/workflows/framework-streamlit.yml' + - 'framework/streamlit/**' + - '/requirements.txt' + push: + branches: [ main ] + paths: + - '.github/workflows/framework-streamlit.yml' + - 'framework/streamlit/**' + - '/requirements.txt' + + # Allow job to be triggered manually. + workflow_dispatch: + + # Run job each night after CrateDB nightly has been published. + schedule: + - cron: '0 3 * * *' + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + test: + name: " + Python: ${{ matrix.python-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ 'ubuntu-latest' ] + python-version: [ '3.10', '3.11', '3.12' ] + cratedb-version: [ 'nightly' ] + + services: + cratedb: + image: crate/crate:${{ matrix.cratedb-version }} + ports: + - 4200:4200 + - 5432:5432 + env: + CRATE_HEAP_SIZE: 4g + + steps: + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + cache: 'pip' + cache-dependency-path: | + requirements.txt + framework/streamlit/requirements.txt + framework/streamlit/requirements-dev.txt + + - name: Install utilities + run: | + pip install -r requirements.txt + + - name: Validate framework/streamlit + run: | + ngr test --accept-no-venv framework/streamlit diff --git a/framework/streamlit/README.md b/framework/streamlit/README.md index 665ff814..5569b976 100644 --- a/framework/streamlit/README.md +++ b/framework/streamlit/README.md @@ -32,3 +32,20 @@ streamlit run basic_sys_summits.py Enjoy the list of mountains. ![image](https://github.com/crate/cratedb-examples/assets/453543/7dc54224-06d0-4cfb-a5e0-b216c03bf3d2) + + +## Development + +Acquire `cratedb-example` repository, and set up sandbox: +```shell +git clone https://github.com/crate/cratedb-examples +cd cratedb-examples +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +Then, invoke the integration test cases: +```shell +ngr test framework/streamlit +``` diff --git a/framework/streamlit/pyproject.toml b/framework/streamlit/pyproject.toml new file mode 100644 index 00000000..8cd2294a --- /dev/null +++ b/framework/streamlit/pyproject.toml @@ -0,0 +1,12 @@ +[tool.pytest.ini_options] +minversion = "2.0" +addopts = """ + -rfEX -p pytester --strict-markers --verbosity=3 + --capture=no + """ +log_level = "DEBUG" +log_cli_level = "DEBUG" +testpaths = ["*.py"] +xfail_strict = true +markers = [ +] diff --git a/framework/streamlit/requirements-dev.txt b/framework/streamlit/requirements-dev.txt new file mode 100644 index 00000000..508a3d0d --- /dev/null +++ b/framework/streamlit/requirements-dev.txt @@ -0,0 +1 @@ +pytest<9 diff --git a/framework/streamlit/test.py b/framework/streamlit/test.py new file mode 100644 index 00000000..7ba5a579 --- /dev/null +++ b/framework/streamlit/test.py @@ -0,0 +1,29 @@ +from streamlit.testing.v1 import AppTest + + +def test_read_sys_summits(): + """ + Verify reading CrateDB's built-in `sys.summits` database table through Streamlit. + """ + + # Invoke Streamlit application. + at = AppTest.from_file("basic_sys_summits.py").run() + + # There should not be an exception. + assert not at.exception + + # Probe dataframe output. + df = at.dataframe.values[0] + mountains = df["mountain"].values + assert "Mont Blanc" in mountains + assert "Matterhorn" in mountains + + # Probe table output. + df = at.table.values[0] + mountains = df["mountain"].values + assert "Mont Blanc" in mountains + assert "Matterhorn" in mountains + + # Probe Markdown output. + assert "Mont Blanc" in at.markdown.values + assert "Monte Rosa" in at.markdown.values From 1a84a1932d427c684088757b1b2bed4cac3e41f3 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 26 Jun 2024 23:16:44 +0200 Subject: [PATCH 3/3] Framework/Streamlit: Add information about connecting to CrateDB Cloud --- .../.streamlit/secrets-cratedb-cloud.toml | 13 +++++++++++++ framework/streamlit/.streamlit/secrets.toml | 2 ++ framework/streamlit/README.md | 15 +++++++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 framework/streamlit/.streamlit/secrets-cratedb-cloud.toml diff --git a/framework/streamlit/.streamlit/secrets-cratedb-cloud.toml b/framework/streamlit/.streamlit/secrets-cratedb-cloud.toml new file mode 100644 index 00000000..3cde509e --- /dev/null +++ b/framework/streamlit/.streamlit/secrets-cratedb-cloud.toml @@ -0,0 +1,13 @@ +# Streamlit secrets file for CrateDB Cloud. +# https://docs.streamlit.io/develop/api-reference/connections/secrets.toml +# .streamlit/secrets.toml + +[connections.cratedb] +dialect = "crate" +host = "example.gke1.us-central1.gcp.cratedb.net" +port = "4200" +username = "admin" +password = "g_,8.F0fNbVSk0.*!n54S5c," + +[connections.cratedb.create_engine_kwargs.connect_args] +ssl = true diff --git a/framework/streamlit/.streamlit/secrets.toml b/framework/streamlit/.streamlit/secrets.toml index f939975d..0eeb42ae 100644 --- a/framework/streamlit/.streamlit/secrets.toml +++ b/framework/streamlit/.streamlit/secrets.toml @@ -1,3 +1,5 @@ +# Streamlit secrets file for CrateDB on localhost. +# https://docs.streamlit.io/develop/api-reference/connections/secrets.toml # .streamlit/secrets.toml [connections.cratedb] diff --git a/framework/streamlit/README.md b/framework/streamlit/README.md index 5569b976..e59fc443 100644 --- a/framework/streamlit/README.md +++ b/framework/streamlit/README.md @@ -9,6 +9,8 @@ Configure database connection address and credentials within make sure to use valid credentials matching your environment. ## Usage + +### CrateDB on localhost To start a CrateDB instance on your machine, invoke: ```shell docker run -it --rm \ @@ -17,6 +19,15 @@ docker run -it --rm \ crate:latest -Cdiscovery.type=single-node ``` +### CrateDB Cloud +Please have a look at the [secrets-cratedb-cloud.toml](.streamlit/secrets-cratedb-cloud.toml) +file as a blueprint. It includes a configuration snippet that is essential for +connecting to CrateDB Cloud. +```toml +[connections.cratedb.create_engine_kwargs.connect_args] +ssl = true +``` + Install dependencies. ```shell pip install -r requirements.txt @@ -36,7 +47,7 @@ Enjoy the list of mountains. ## Development -Acquire `cratedb-example` repository, and set up sandbox: +Acquire `cratedb-example` repository, and set up development sandbox. ```shell git clone https://github.com/crate/cratedb-examples cd cratedb-examples @@ -45,7 +56,7 @@ source .venv/bin/activate pip install -r requirements.txt ``` -Then, invoke the integration test cases: +Invoke the integration test cases. ```shell ngr test framework/streamlit ```