From 53f85b259c5ede534dae366471aa97c6a2142bf2 Mon Sep 17 00:00:00 2001 From: Aleksandr Movchan Date: Mon, 9 Dec 2024 20:24:46 +0000 Subject: [PATCH] Add Snowflake DB support --- aana/alembic/env.py | 6 + aana/alembic/versions/5ad873484aa3_init.py | 8 +- aana/configs/db.py | 26 ++- aana/storage/custom_types.py | 60 ++++++ aana/storage/models/task.py | 8 +- aana/storage/op.py | 76 +++++++- poetry.lock | 205 ++++++++++++++++++++- pyproject.toml | 2 + 8 files changed, 374 insertions(+), 17 deletions(-) create mode 100644 aana/storage/custom_types.py diff --git a/aana/alembic/env.py b/aana/alembic/env.py index 499c7216..7f5e658f 100644 --- a/aana/alembic/env.py +++ b/aana/alembic/env.py @@ -1,11 +1,17 @@ from logging.config import fileConfig from alembic import context +from alembic.ddl.impl import DefaultImpl from sqlalchemy import engine_from_config, pool from aana.configs.settings import settings from aana.storage.models.base import BaseEntity + +class SnowflakeImpl(DefaultImpl): + __dialect__ = "snowflake" + + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config diff --git a/aana/alembic/versions/5ad873484aa3_init.py b/aana/alembic/versions/5ad873484aa3_init.py index 89833429..ce9fc408 100644 --- a/aana/alembic/versions/5ad873484aa3_init.py +++ b/aana/alembic/versions/5ad873484aa3_init.py @@ -10,6 +10,8 @@ import sqlalchemy as sa from alembic import op +from aana.storage.custom_types import JSON, VARIANT + # revision identifiers, used by Alembic. revision: str = "5ad873484aa3" down_revision: str | None = None @@ -90,7 +92,7 @@ def upgrade() -> None: ) op.create_table( "tasks", - sa.Column("id", sa.UUID(), nullable=False, comment="Task ID"), + sa.Column("id", sa.String(), nullable=False, comment="Task ID"), sa.Column( "endpoint", sa.String(), @@ -140,7 +142,7 @@ def upgrade() -> None: ), sa.Column( "result", - sa.JSON(), + JSON(), nullable=True, comment="Result of the task in JSON format", ), @@ -176,7 +178,7 @@ def upgrade() -> None: comment="Full text transcript of media", ), sa.Column( - "segments", sa.JSON(), nullable=False, comment="Segments of the transcript" + "segments", JSON(), nullable=False, comment="Segments of the transcript" ), sa.Column( "language", diff --git a/aana/configs/db.py b/aana/configs/db.py index dd56f1ab..bfb01b55 100644 --- a/aana/configs/db.py +++ b/aana/configs/db.py @@ -35,12 +35,34 @@ class PostgreSQLConfig(TypedDict): database: str +class SnowflakeConfig(TypedDict): + """Config values for Snowflake. + + Attributes: + account (str): The account name. + user (str): The user to connect to the Snowflake server. + password (str): The password to connect to the Snowflake server. + database (str): The database name. + schema (str): The schema name. + warehouse (str): The warehouse name. + role (str): The role name. + """ + + account: str + user: str + password: str + database: str + schema: str + warehouse: str + role: str + + class DbSettings(BaseSettings): """Database configuration. Attributes: datastore_type (DbType | str): The type of the datastore. Default is DbType.SQLITE. - datastore_config (SQLiteConfig | PostgreSQLConfig): The configuration for the datastore. + datastore_config (SQLiteConfig | PostgreSQLConfig | SnowflakeConfig): The configuration for the datastore. Default is SQLiteConfig(path="/var/lib/aana_data"). pool_size (int): The number of connections to keep in the pool. Default is 5. max_overflow (int): The number of connections that can be created when the pool is exhausted. @@ -50,7 +72,7 @@ class DbSettings(BaseSettings): """ datastore_type: DbType | str = DbType.SQLITE - datastore_config: SQLiteConfig | PostgreSQLConfig = SQLiteConfig( + datastore_config: SQLiteConfig | PostgreSQLConfig | SnowflakeConfig = SQLiteConfig( path="/var/lib/aana_data" ) pool_size: int = 5 diff --git a/aana/storage/custom_types.py b/aana/storage/custom_types.py new file mode 100644 index 00000000..11afcdbd --- /dev/null +++ b/aana/storage/custom_types.py @@ -0,0 +1,60 @@ +import orjson +from snowflake.sqlalchemy.custom_types import VARIANT as SnowflakeVariantType +from sqlalchemy import func +from sqlalchemy.types import JSON as SqlAlchemyJSON +from sqlalchemy.types import TypeDecorator + + +class VARIANT(SnowflakeVariantType): + """Extends VARIANT type for better SqlAlchemy support.""" + + def bind_expression(self, bindvalue): + """Wraps value with PARSE_JSON for Snowflake.""" + return func.PARSE_JSON(bindvalue) + + def result_processor(self, dialect, coltype): + """Convert JSON string to Python dictionary when retrieving.""" + + def process(value): + if value is None: + return None + try: + return orjson.loads(value) + except (ValueError, TypeError): + return value # Return raw value if not valid JSON + + return process + + +JSON = VARIANT + +# class JSON(TypeDecorator): +# """Custom JSON type that supports Snowflake-specific and standard dialects.""" + +# impl = SqlAlchemyJSON # Default to standard SQLAlchemy JSON + +# def load_dialect_impl(self, dialect): +# """Load dialect-specific implementation.""" +# if dialect.name == "snowflake": +# return SnowflakeVariantType() +# return self.impl + +# def bind_expression(self, bindvalue): +# """Handle binding expressions dynamically.""" +# if hasattr( +# bindvalue.type, "bind_expression" +# ): # Check if impl has bind_expression +# return bindvalue.type.bind_expression(bindvalue) +# return bindvalue # Default binding behavior + +# def process_result_value(self, value, dialect): +# """Process the result based on dialect.""" +# if dialect.name == "snowflake": +# if value is None: +# return None +# try: +# return orjson.loads(value) +# except (ValueError, TypeError): +# return value # Return raw value if not valid JSON +# # For other dialects, call the default implementation +# return self.impl.process_result_value(value, dialect) diff --git a/aana/storage/models/task.py b/aana/storage/models/task.py index c5b15212..dd437670 100644 --- a/aana/storage/models/task.py +++ b/aana/storage/models/task.py @@ -1,13 +1,19 @@ +import re import uuid from enum import Enum from sqlalchemy import ( - JSON, UUID, PickleType, + event, + insert, + select, ) +from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.sql import Insert +from aana.storage.custom_types import JSON from aana.storage.models.base import BaseEntity, TimeStampEntity, timestamp diff --git a/aana/storage/op.py b/aana/storage/op.py index 9dc265f5..61564455 100644 --- a/aana/storage/op.py +++ b/aana/storage/op.py @@ -1,3 +1,4 @@ +import json import typing from enum import Enum from pathlib import Path @@ -5,7 +6,8 @@ import orjson from alembic import command from alembic.config import Config -from sqlalchemy import create_engine +from snowflake.sqlalchemy import URL as SNOWFLAKE_URL +from sqlalchemy import create_engine, event from aana.exceptions.runtime import EmptyMigrationsException from aana.utils.core import get_module_dir @@ -14,6 +16,12 @@ if typing.TYPE_CHECKING: from aana.configs.db import DbSettings +import re + +from snowflake.sqlalchemy.custom_types import OBJECT, VARIANT +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.sql import Insert + class DbType(str, Enum): """Engine types for relational database. @@ -25,6 +33,7 @@ class DbType(str, Enum): POSTGRESQL = "postgresql" SQLITE = "sqlite" + SNOWFLAKE = "snowflake" def create_postgresql_engine(db_config: "DbSettings"): @@ -76,6 +85,69 @@ def create_sqlite_engine(db_config: "DbSettings"): ) +def create_snowflake_engine(db_config: "DbSettings"): + """Create a Snowflake SQLAlchemy engine based on the provided configuration. + + Args: + db_config (DbSettings): Database configuration. + + Returns: + sqlalchemy.engine.Engine: SQLAlchemy engine instance. + """ + datastore_config = db_config.datastore_config + connection_string = SNOWFLAKE_URL(**datastore_config) + engine = create_engine( + connection_string, + pool_size=db_config.pool_size, + max_overflow=db_config.max_overflow, + pool_recycle=db_config.pool_recycle, + ) + + @event.listens_for(engine, "before_cursor_execute") + def preprocess_parameters( + conn, cursor, statement, parameters, context, executemany + ): + """Preprocess parameters before executing a query.""" + if isinstance(parameters, dict): # Handle dict-style parameters + for key, value in parameters.items(): + # Convert VARIANT type to JSON string + if ( + isinstance(value, dict) + and context.compiled + and key in context.compiled.binds + and isinstance(context.compiled.binds["result"].type, VARIANT) + ): + parameters[key] = jsonify(value) + + @compiles(Insert, "default") + def compile_insert(insert_stmt, compiler, **kwargs): + """Compile INSERT statements to use SELECT instead of VALUES for Snowflake PARSE_JSON.""" + sql = compiler.visit_insert(insert_stmt, **kwargs) + + # Only transform if PARSE_JSON is present in the SQL + if "PARSE_JSON" not in sql: + return sql + + # Locate the VALUES clause and replace it + def replace_values_with_select(sql): + # Regex to find `VALUES (...)` ensuring balanced parentheses + pattern = r"VALUES\s*(\((?:[^)(]+|\((?:[^)(]+|\([^)(]*\))*\))*\))" + match = re.search(pattern, sql) + if match: + values_clause = match.group(1) # Captures the `(...)` after VALUES + # Replace VALUES (...) with SELECT ... + return sql.replace( + f"VALUES {values_clause}", f"SELECT {values_clause[1:-1]}" + ) + return sql + + # Replace the VALUES clause with SELECT + sql = replace_values_with_select(sql) + return sql + + return engine + + def create_database_engine(db_config: "DbSettings"): """Create SQLAlchemy engine based on the provided configuration. @@ -91,6 +163,8 @@ def create_database_engine(db_config: "DbSettings"): return create_postgresql_engine(db_config) elif db_type == DbType.SQLITE: return create_sqlite_engine(db_config) + elif db_type == DbType.SNOWFLAKE: + return create_snowflake_engine(db_config) else: raise ValueError(f"Unsupported database type: {db_type}") # noqa: TRY003 diff --git a/poetry.lock b/poetry.lock index 5ffafa43..5f3a94f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -241,6 +241,17 @@ files = [ {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, ] +[[package]] +name = "asn1crypto" +version = "1.5.1" +description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +optional = false +python-versions = "*" +files = [ + {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, + {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, +] + [[package]] name = "asteroid-filterbanks" version = "0.4.0" @@ -874,6 +885,57 @@ files = [ dev = ["flake8 (>=4.0.1)", "parameterized", "pylint (>=2.11.0)", "pytest", "pytest-cov", "pytest-timeout", "setuptools", "testfixtures", "tox (<5.0.0)"] test = ["parameterized", "pytest", "pytest-cov", "pytest-timeout", "testfixtures", "tox (<5.0.0)"] +[[package]] +name = "cryptography" +version = "44.0.0" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.7" +files = [ + {file = "cryptography-44.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385"}, + {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, + {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, + {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, + {file = "cryptography-44.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd"}, + {file = "cryptography-44.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba"}, + {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, + {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, + {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, + {file = "cryptography-44.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa"}, + {file = "cryptography-44.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c"}, + {file = "cryptography-44.0.0.tar.gz", hash = "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0)"] +docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +sdist = ["build (>=1.0.0)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi (>=2024)", "cryptography-vectors (==44.0.0)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "ctranslate2" version = "4.5.0" @@ -1547,8 +1609,8 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" proto-plus = [ - {version = ">=1.22.3,<2.0.0dev", markers = "python_version < \"3.13\""}, {version = ">=1.25.0,<2.0.0dev", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0dev", markers = "python_version < \"3.13\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -3422,9 +3484,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.21.2", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.23.3", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\" and python_version < \"3.13\""}, + {version = ">=1.23.3", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, + {version = ">=1.21.2", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] [package.extras] @@ -4197,10 +4259,10 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] [[package]] @@ -4413,9 +4475,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -5350,8 +5412,8 @@ files = [ annotated-types = ">=0.6.0" pydantic-core = "2.23.4" typing-extensions = [ - {version = ">=4.6.1", markers = "python_version < \"3.13\""}, {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, ] [package.extras] @@ -5493,6 +5555,23 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyjwt" +version = "2.10.1" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pymdown-extensions" version = "10.12" @@ -5511,6 +5590,24 @@ pyyaml = "*" [package.extras] extra = ["pygments (>=2.12)"] +[[package]] +name = "pyopenssl" +version = "24.3.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"}, + {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"}, +] + +[package.dependencies] +cryptography = ">=41.0.5,<45" + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] +test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] + [[package]] name = "pyparsing" version = "3.2.0" @@ -6027,8 +6124,8 @@ grpcio = ">=1.41.0" grpcio-tools = ">=1.41.0" httpx = {version = ">=0.20.0", extras = ["http2"]} numpy = [ - {version = ">=1.21", markers = "python_version >= \"3.8\" and python_version < \"3.12\""}, {version = ">=1.26", markers = "python_version >= \"3.12\""}, + {version = ">=1.21", markers = "python_version >= \"3.8\" and python_version < \"3.12\""}, ] portalocker = ">=2.7.0,<3.0.0" pydantic = ">=1.10.8" @@ -7000,6 +7097,83 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "snowflake-connector-python" +version = "3.12.4" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake_connector_python-3.12.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f141c159e3244bd660279f87f32e39351b2845fcb75f8138f31d2219f983b05"}, + {file = "snowflake_connector_python-3.12.4-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:091458ba777c24adff659c5c28f0f5bb0bcca8a9b6ecc5641ae25b7c20a8f43d"}, + {file = "snowflake_connector_python-3.12.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23049d341da681ec7131cead71cdf7b1761ae5bcc08bcbdb931dcef6c25e8a5f"}, + {file = "snowflake_connector_python-3.12.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc88a09d77a8ce7e445094b2409b606ddb208b5fc9f7c7a379d0255a8d566e9d"}, + {file = "snowflake_connector_python-3.12.4-cp310-cp310-win_amd64.whl", hash = "sha256:3c33fbba036805c1767ea48eb40ffc3fb79d61f2a4bb4e77b571ea6f6a998be8"}, + {file = "snowflake_connector_python-3.12.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ec5cfaa1526084cf4d0e7849d5ace601245cb4ad9675ab3cd7d799b3abea481"}, + {file = "snowflake_connector_python-3.12.4-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:ff225824b3a0fa5e822442de72172f97028f04ae183877f1305d538d8d6c5d11"}, + {file = "snowflake_connector_python-3.12.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9beced2789dc75e8f1e749aa637e7ec9b03302b4ed4b793ae0f1ff32823370e"}, + {file = "snowflake_connector_python-3.12.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ea47450a04ff713f3adf28053e34103bd990291e62daee9721c76597af4b2b5"}, + {file = "snowflake_connector_python-3.12.4-cp311-cp311-win_amd64.whl", hash = "sha256:748f9125854dca07ea471bb2bb3c5bb932a53f9b8a77ba348b50b738c77203ce"}, + {file = "snowflake_connector_python-3.12.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bcd0371b20d199f15e6a3c0b489bf18e27f2a88c84cf3194b2569ca039fa7d1"}, + {file = "snowflake_connector_python-3.12.4-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:7900d82a450b206fa2ed6c42cd65d9b3b9fd4547eca1696937175fac2a03ba37"}, + {file = "snowflake_connector_python-3.12.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:300f0562aeea55e40ee03b45205dbef7b78f5ba2f1787a278c7b807e7d8db22c"}, + {file = "snowflake_connector_python-3.12.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6762a00948f003be55d7dc5de9de690315d01951a94371ec3db069d9303daba"}, + {file = "snowflake_connector_python-3.12.4-cp312-cp312-win_amd64.whl", hash = "sha256:83ca896790a7463b6c8cd42e1a29b8ea197cc920839ae6ee96a467475eab4ec2"}, + {file = "snowflake_connector_python-3.12.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:886d2cbf0aaa5eac81df05efefaff145773e732d88ded7f6e2465580a4c31d1e"}, + {file = "snowflake_connector_python-3.12.4-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:c06f553486ba6715f18bb2933446adace4cfbb40f54b7fa0d46839930ecf945a"}, + {file = "snowflake_connector_python-3.12.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:158f616e36453e72550937256b160badcea6d4fb38bff21fdf551813ebe409b4"}, + {file = "snowflake_connector_python-3.12.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b80d2e3233419920f116e277ad8d422fbbce172c25209acf9095fdde6293b0"}, + {file = "snowflake_connector_python-3.12.4-cp38-cp38-win_amd64.whl", hash = "sha256:f40bd66199064b73d3b313132d3a8c297aedc9da1489e38bfa2e925d17871f6d"}, + {file = "snowflake_connector_python-3.12.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34947e4147d65b9211ab785f76c16cfb2f5d5a0b639358f72ccbf524ff3eda6f"}, + {file = "snowflake_connector_python-3.12.4-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:de703efc36b9b12eab6d75dffeec362b0bbcc2f33ffd9913b0d178d2bf30d8cd"}, + {file = "snowflake_connector_python-3.12.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5010cee09cfe20f72dbd9b1f3ee21d07fd280c03db243aa2c048708e7331f358"}, + {file = "snowflake_connector_python-3.12.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28be3b68e2b1d880373506eab4bae5b323d7de6215630585435ab60d3fea7feb"}, + {file = "snowflake_connector_python-3.12.4-cp39-cp39-win_amd64.whl", hash = "sha256:bc8766d16c98a75a57e65fb7697c6732b871309c2fa79698b54fb18cbd36733c"}, + {file = "snowflake_connector_python-3.12.4.tar.gz", hash = "sha256:289e0691dfbf8ec8b7a8f58bcbb95a819890fe5e5b278fdbfc885059a63a946f"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +certifi = ">=2017.4.17" +cffi = ">=1.9,<2.0.0" +charset_normalizer = ">=2,<4" +cryptography = ">=3.1.0" +filelock = ">=3.5,<4" +idna = ">=2.5,<4" +packaging = "*" +platformdirs = ">=2.6.0,<5.0.0" +pyjwt = "<3.0.0" +pyOpenSSL = ">=22.0.0,<25.0.0" +pytz = "*" +requests = "<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing_extensions = ">=4.3,<5" + +[package.extras] +development = ["Cython", "coverage", "more-itertools", "numpy (<1.27.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.5.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<3.0.0)", "pyarrow"] +secure-local-storage = ["keyring (>=23.1.0,<26.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.7.1" +description = "Snowflake SQLAlchemy Dialect" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake_sqlalchemy-1.7.1-py3-none-any.whl", hash = "sha256:eecb63e6830e7fec2a0fc5c583c0e9903fe1b2ea40bcac974e03932cd24662f2"}, + {file = "snowflake_sqlalchemy-1.7.1.tar.gz", hash = "sha256:a06b78d8b83ca74318e6fbb2982b9fbd9ce99e202f502c1f6af7ba69d05da1f5"}, +] + +[package.dependencies] +snowflake-connector-python = "<4.0.0" +sqlalchemy = ">=1.4.19" + +[package.extras] +development = ["mock", "numpy", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytz", "syrupy (==4.6.1)"] +pandas = ["snowflake-connector-python[pandas]"] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -7567,6 +7741,17 @@ files = [ {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, ] +[[package]] +name = "tomlkit" +version = "0.13.2" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] + [[package]] name = "torch" version = "2.4.0" @@ -8730,4 +8915,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "fc8559a51bc4a0a2fe78d7ad74f2463c8cb5f444459c1ef1009a9f105d5f2cf2" +content-hash = "39a57a061843cd3802155ee7d206ab50d86b8c18f4d1b9d16cf60de1ca3adeec" diff --git a/pyproject.toml b/pyproject.toml index d3bf384d..2c178a67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ ray = {extras = ["serve"], version = ">=2.20"} rapidfuzz = "^3.4.0" scipy = "^1.11.3" sentence-transformers = ">=2.6.1" +snowflake-sqlalchemy = "^1.7.1" sqlalchemy = {extras = ["mypy"], version = "^2.0.23"} transformers = ">=4.44.2" torch = "^2.4.0" @@ -57,6 +58,7 @@ vllm = ">=0.6.1, <0.6.4" # See https://github.com/vllm-project/vllm/issues/10451 yt-dlp = ">=2024.08.06" outlines = {extras = ["serve"], version = "^0.0.46"} + [tool.poetry.group.dev.dependencies] ipykernel = "^6.25.2" matplotlib = "^3.8.2"