Upgrade dependencies to fix various vulnerabilities (#48)

app/dl_control_api/pyproject.toml

@@ -10,7 +10,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = ">=3.10, <3.12"
-Flask = ">=2.0.3"
+Flask = ">=2.2.5"
 blinker = ">=1.5"
 raven = ">=6.10.0"
 datalens-version = {path = "../../lib/dl_version"}

lib/dl_api_commons/pyproject.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 aiohttp = ">=3.8.1"
 attrs = ">=22.2.0"
-flask = ">=2.0.3"
+flask = ">=2.2.5"
 ipdb = ">=0.13.13"
 marshmallow = ">=3.19.0"
 multidict = ">=4.0"

lib/dl_api_lib/dl_api_lib/api_decorators.py

@@ -86,7 +86,7 @@ def decorator(f):  # type: ignore  # TODO: fix
 
         @wraps(f)
         def wrapper(*args, **kwargs):  # type: ignore  # TODO: fix
-            body = request.get_json()
+            body = request.get_json() if body_schema is not None else None
 
             if LOGGER.isEnabledFor(logging.INFO):
                 dbg_body_data = mask_sensitive_fields_by_name_in_json_recursive(body)

lib/dl_api_lib/pyproject.toml

@@ -11,7 +11,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 aiohttp = ">=3.8.1"
 attrs = ">=22.2.0"
-flask = ">=2.0.3"
+flask = ">=2.2.5"
 flask-marshmallow = ">=0.14.0"
 flask-restx = ">=1.1.0"
 marshmallow = ">=3.19.0"
@@ -20,7 +20,7 @@ more-itertools = ">=9.1.0"
 pandas = ">=1.5.3"
 python = ">=3.10, <3.12"
 sentry-sdk = ">=1.15.0"
-werkzeug = ">=2.0.3"
+werkzeug = ">=2.2.3"
 statcommons = {path = "../statcommons"}
 datalens-api-commons = {path = "../dl_api_commons"}
 datalens-utils = {path = "../dl_utils"}

lib/dl_api_lib_testing/pyproject.toml

@@ -12,10 +12,10 @@ readme = "README.md"
 [tool.poetry.dependencies]
 aiohttp = ">=3.8.1"
 attrs = ">=22.2.0"
-flask = ">=2.0.3"
+flask = ">=2.2.5"
 pytest = ">=7.2.2"
 python = ">=3.10, <3.12"
-werkzeug = ">=2.0.3"
+werkzeug = ">=2.2.3"
 datalens-api-commons = {path = "../dl_api_commons"}
 datalens-utils = {path = "../dl_utils"}
 datalens-constants = {path = "../dl_constants"}

lib/dl_configs/pyproject.toml

@@ -10,10 +10,10 @@ license = "Apache 2.0"
 
 [tool.poetry.dependencies]
 attrs = ">=22.2.0"
-cryptography = ">=3.3.2"
+cryptography = ">=41.0.4"
 dynamic-enum = {path = "../dynamic_enum"}
 python = ">=3.10, <3.12"
-pyyaml = ">=5.3.1"
+pyyaml = ">=6.0.1"
 typeguard = ">=2.13.3"
 datalens-utils = {path = "../dl_utils"}
 datalens-constants = {path = "../dl_constants"}

lib/dl_connector_bitrix_gds/pyproject.toml

@@ -10,7 +10,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 attrs = ">=22.2.0"
 marshmallow = ">=3.19.0"
-redis = ">=4.5.1"
+redis = ">=4.5.4"
 sqlalchemy = ">=1.4.46, <2.0"
 python = ">=3.10, <3.12"
 redis_cache_lock = {path = "../redis-cache-lock"}

lib/dl_connector_snowflake/dl_connector_snowflake_tests/conftest.py

@@ -0,0 +1 @@
+pytest_plugins = ("aiohttp.pytest_plugin",)  # and it, in turn, includes 'pytest_asyncio.plugin'

lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/api/base.py

@@ -4,7 +4,10 @@
 
 from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration
 from dl_api_lib_testing.connection_base import ConnectionTestBase
-from dl_api_lib_testing.data_api_base import StandardizedDataApiTestBase
+from dl_api_lib_testing.data_api_base import (
+    DataApiTestParams,
+    StandardizedDataApiTestBase,
+)
 from dl_api_lib_testing.dataset_base import DatasetTestBase
 
 from dl_connector_snowflake.core.constants import (
@@ -60,4 +63,14 @@ def dataset_params(self, sf_secrets) -> dict:
 
 
 class SnowFlakeDataApiTestBase(SnowFlakeDatasetTestBase, StandardizedDataApiTestBase):
-    pass
+    bi_compeng_pg_on = False
+
+    @pytest.fixture(scope="class")
+    def data_api_test_params(self) -> DataApiTestParams:
+        return DataApiTestParams(
+            two_dims=("Category", "City"),
+            summable_field="Sales",
+            range_field="Sales",
+            distinct_field="City",
+            date_field="Order Date",
+        )

lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/conftest.py

@@ -2,7 +2,6 @@
 
 import pytest
 
-from dl_api_lib.loader import load_api_lib
 from dl_api_lib_testing.initialization import initialize_api_lib_test
 from dl_formula_testing.forced_literal import forced_literal_use
 from dl_testing.env_params.generic import GenericEnvParamGetter
@@ -11,9 +10,6 @@
 from dl_connector_snowflake_tests.ext.config import API_TEST_CONFIG
 
 
-pytest_plugins = ("aiohttp.pytest_plugin",)  # and it, in turn, includes 'pytest_asyncio.plugin'
-
-
 def pytest_configure(config):  # noqa
     initialize_api_lib_test(pytest_config=config, api_test_config=API_TEST_CONFIG)
 

lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/core/base.py

@@ -1,5 +1,9 @@
+import asyncio
 import datetime
-from typing import Any
+from typing import (
+    Any,
+    Generator,
+)
 
 import pytest
 
@@ -19,6 +23,15 @@ class BaseSnowFlakeTestClass(BaseConnectionTestClass[ConnectionSQLSnowFlake]):
     core_test_config = test_config.CORE_TEST_CONFIG
     engine_config_cls = SnowFlakeDbEngineConfig
 
+    @pytest.fixture(autouse=True)
+    # FIXME: This fixture is a temporary solution for failing core tests when they are run together with api tests
+    def loop(self, event_loop: asyncio.AbstractEventLoop) -> Generator[asyncio.AbstractEventLoop, None, None]:
+        asyncio.set_event_loop(event_loop)
+        yield event_loop
+        # Attempt to cover an old version of pytest-asyncio:
+        # https://github.com/pytest-dev/pytest-asyncio/commit/51d986cec83fdbc14fa08015424c79397afc7ad9
+        asyncio.set_event_loop_policy(None)
+
     @pytest.fixture(scope="class")
     def db_url(self) -> str:
         return test_config.DB_DSN

lib/dl_connector_snowflake/pyproject.toml

@@ -14,7 +14,7 @@ attrs = ">=22.2.0"
 marshmallow = ">=3.19.0"
 python = ">=3.10, <3.12"
 requests = ">=2.28.2"
-snowflake-connector-python = ">=2.9.0"
+snowflake-connector-python = ">=3.1.0"
 snowflake-sqlalchemy = ">=1.4.6"
 sqlalchemy = ">=1.4.46, <2.0"
 datalens-api-commons = {path = "../dl_api_commons"}

lib/dl_core/dl_core/connection_executors/remote_query_executor/app_async.py

@@ -7,7 +7,6 @@
 from typing import (
     TYPE_CHECKING,
     Any,
-    Optional,
     Type,
     Union,
 )

lib/dl_core/dl_core/connectors/settings/registry.py

@@ -17,7 +17,7 @@ def register_connector_settings_class(
     fallback: SettingsFallbackType,
 ) -> None:
     if (registered_settings_class := CONNECTORS_SETTINGS_CLASSES.get(conn_type)) is not None:
-        assert registered_settings_class == settings_class
+        assert registered_settings_class == settings_class, f"{registered_settings_class} vs {settings_class}"
         assert CONNECTORS_SETTINGS_FALLBACKS[conn_type] == fallback
     else:
         assert conn_type not in CONNECTORS_SETTINGS_FALLBACKS

lib/dl_core/pyproject.toml

@@ -10,8 +10,8 @@ readme = "README.md"
 
 
 [tool.poetry.dependencies]
-Flask = ">=2.0.3"
-Werkzeug = ">=2.0.3"
+Flask = ">=2.2.5"
+Werkzeug = ">=2.2.3"
 aiobotocore = ">=2.4.2"
 aiodns = ">=3.0.0"
 aiohttp = ">=3.8.1"
@@ -20,9 +20,9 @@ arq = ">=0.25.0"
 attrs = ">=22.2.0"
 boto3 = ">=1.24.59"
 botocore = ">=1.27.59,<1.27.60"
-cryptography = ">=3.3.2"
+cryptography = ">=41.0.4"
 dnspython = ">=2.2.1"
-flask = ">=2.0.3"
+flask = ">=2.2.5"
 grpcio = ">=1.58.0"
 jaeger-client = ">=4.8.0"
 lz4 = ">=4.3.2"
@@ -33,15 +33,15 @@ opentracing = ">=2.4.0"
 python = ">=3.10, <3.12"
 python-dateutil = ">=2.8.2"
 pytz = ">=2022.7.1"
-pyyaml = ">=5.3.1"
+pyyaml = ">=6.0.1"
 raven = ">=6.10.0"
-redis = ">=4.5.1"
+redis = ">=4.5.4"
 requests = ">=2.28.2"
 shortuuid = ">=1.0.11"
 sqlalchemy = ">=1.4.46, <2.0"
 typing-extensions = ">=4.5.0"
-uwsgi = ">=2.0.21"
-werkzeug = ">=2.0.3"
+uwsgi = ">=2.0.22"
+werkzeug = ">=2.2.3"
 xxhash = ">=3.2.0"
 # Check if really required
 clickhouse-sqlalchemy = {path = "../clickhouse-sqlalchemy"}

lib/dl_core_testing/pyproject.toml

@@ -10,8 +10,8 @@ readme = "README.md"
 
 
 [tool.poetry.dependencies]
-Flask = ">=2.0.3"
-Werkzeug = ">=2.0.3"
+Flask = ">=2.2.5"
+Werkzeug = ">=2.2.3"
 aiohttp = ">=3.8.1"
 attrs = ">=22.2.0"
 marshmallow = ">=3.19.0"
@@ -21,7 +21,7 @@ python = ">=3.10, <3.12"
 requests = ">=2.28.2"
 shortuuid = ">=1.0.11"
 sqlalchemy = ">=1.4.46, <2.0"
-uwsgi = ">=2.0.21"
+uwsgi = ">=2.0.22"
 statcommons = {path = "../statcommons"}
 datalens-api-commons = {path = "../dl_api_commons"}
 datalens-utils = {path = "../dl_utils"}

lib/dl_file_secure_reader_lib/pyproject.toml

@@ -11,7 +11,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 aiohttp = ">=3.8.1"
-certifi = ">=2022.12.7"
+certifi = ">=2023.7.22"
 openpyxl = ">=3.0.10"
 python = ">=3.10, <3.12"
 

lib/dl_file_uploader_api_lib/pyproject.toml

@@ -18,7 +18,7 @@ gunicorn = ">=20.1.0"
 marshmallow = ">=3.19.0"
 marshmallow-oneofschema = ">=3.0.1"
 python = ">=3.10, <3.12"
-redis = ">=4.5.1"
+redis = ">=4.5.4"
 sentry-sdk = ">=1.15.0"
 datalens-file-uploader-task-interface = {path = "../dl_file_uploader_task_interface"}
 datalens-api-commons = {path = "../dl_api_commons"}
@@ -32,7 +32,7 @@ datalens-task-processor = {path = "../dl_task_processor"}
 [tool.poetry.group.tests.dependencies]
 pytest = ">=7.2.2"
 pytest-asyncio = ">=0.20.3"
-uwsgi = ">=2.0.21"
+uwsgi = ">=2.0.22"
 datalens-file-secure-reader-lib = {path = "../dl_file_secure_reader_lib"}
 datalens-testing = {path = "../dl_testing"}
 datalens-core-testing = {path = "../dl_core_testing"}

lib/dl_file_uploader_lib/pyproject.toml

@@ -16,7 +16,7 @@ botocore = ">=1.27.59,<1.27.60"
 marshmallow = ">=3.19.0"
 marshmallow-oneofschema = ">=3.0.1"
 python = ">=3.10, <3.12"
-redis = ">=4.5.1"
+redis = ">=4.5.4"
 ujson = ">=1.35"
 yarl = ">=1.8.2"
 datalens-api-commons = {path = "../dl_api_commons"}

lib/dl_file_uploader_task_interface/pyproject.toml

@@ -26,7 +26,7 @@ datalens-task-processor = {path = "../dl_task_processor"}
 [tool.poetry.group.tests.dependencies]
 pytest = ">=7.2.2"
 pytest-asyncio = ">=0.20.3"
-uwsgi = ">=2.0.21"
+uwsgi = ">=2.0.22"
 datalens-testing = {path = "../dl_testing"}
 datalens-core = {path = "../dl_core"}
 

lib/dl_file_uploader_worker_lib/pyproject.toml

@@ -17,7 +17,7 @@ attrs = ">=22.2.0"
 botocore = ">=1.27.59,<1.27.60"
 cchardet = ">=2.1.7"
 python = ">=3.10, <3.12"
-redis = ">=4.5.1"
+redis = ">=4.5.4"
 shortuuid = ">=1.0.11"
 clickhouse-sqlalchemy = {path = "../clickhouse-sqlalchemy"}
 datalens-file-uploader-task-interface = {path = "../dl_file_uploader_task_interface"}

lib/dl_testing/dl_testing/containers.py

@@ -55,7 +55,7 @@ def get_test_container_hostport(
     else:
         try:
             with open(file_path) as dcyml:
-                docker_compose_yml = yaml.load(dcyml)
+                docker_compose_yml = yaml.safe_load(dcyml)
         except FileNotFoundError:
             if fallback_port is not None:
                 return HostPort(host="127.0.0.1", port=fallback_port)

lib/dl_testing/pyproject.toml

@@ -15,7 +15,7 @@ python-dotenv = ">=1.0.0"
 psycopg2-binary = ">=2.9.6"
 pytest = ">=7.2.2"
 python = ">=3.10, <3.12"
-pyyaml = ">=5.3.1"
+pyyaml = ">=6.0.1"
 datalens-utils = {path = "../dl_utils"}
 
 [tool.poetry.plugins.env_param_getters]

lib/redis-cache-lock/pyproject.toml

@@ -15,7 +15,7 @@ readme = "README.rst"
 [tool.poetry.dependencies]
 attrs = ">=22.2.0"
 python = ">=3.10, <3.12"
-redis = ">=4.5.1"
+redis = ">=4.5.4"
 
 [tool.poetry.group.tests.dependencies]
 [build-system]

lib/statcommons/pyproject.toml

@@ -9,7 +9,7 @@ license = "Apache 2.0"
 
 
 [tool.poetry.dependencies]
-flask = ">=2.0.3"
+flask = ">=2.2.5"
 python = ">=3.10, <3.12"
 [build-system]
 requires = ["poetry-core"]