Implemented function extensions for formula ref and connectorized som…

…e note blocks (#70)

docker_build/target_translations/target_update_po/Dockerfile

@@ -28,6 +28,8 @@ RUN for domain_pair in $(dl-package --package-path=. list-i18n-domains) ; do \
     for locale in $(ls $PACKAGE_NAME/locales/) ; do \
     mkdir -p $LOCALES_DIR/$locale/LC_MESSAGES ; \
     msgmerge $PACKAGE_NAME/locales/$locale/LC_MESSAGES/$DOMAIN_NAME.po $DOMAIN_NAME.po > $LOCALES_DIR/$locale/LC_MESSAGES/$DOMAIN_NAME.po; \
+    # Now replace the date with a fixed one so that it doesn't get updated every time
+    sed -i -r "s/POT-Creation-Date: [0-9]+-[0-9]+-[0-9]+ [0-9]+:[0-9]+\\+[0-9]+/POT-Creation-Date: 2023-09-22 08:16+0000/g" $LOCALES_DIR/$locale/LC_MESSAGES/$DOMAIN_NAME.po; \
     done; \
     done
 

lib/dl_connector_clickhouse/dl_connector_clickhouse/formula_ref/plugin.py

@@ -1,10 +1,22 @@
-from dl_formula_ref.functions.type_conversion import DbCastExtension
+from dl_formula_ref.functions.date import FUNCTION_DATETRUNC
+from dl_formula_ref.functions.operator import FUNCTION_OP_COMPARISON
+from dl_formula_ref.functions.type_conversion import (
+    FUNCTION_DATE,
+    DbCastExtension,
+)
 from dl_formula_ref.plugins.base.plugin import FormulaRefPlugin
+from dl_formula_ref.registry.note import (
+    Note,
+    NoteLevel,
+)
 
 from dl_connector_clickhouse.formula.constants import ClickHouseDialect
 from dl_connector_clickhouse.formula.definitions.functions_type import FuncDbCastClickHouseBase
 from dl_connector_clickhouse.formula_ref.human_dialects import HUMAN_DIALECTS
-from dl_connector_clickhouse.formula_ref.i18n import CONFIGS
+from dl_connector_clickhouse.formula_ref.i18n import (
+    CONFIGS,
+    Translatable,
+)
 
 
 class ClickHouseFormulaRefPlugin(FormulaRefPlugin):
@@ -15,3 +27,45 @@ class ClickHouseFormulaRefPlugin(FormulaRefPlugin):
     db_cast_extension = DbCastExtension(
         type_whitelists=FuncDbCastClickHouseBase.WHITELISTS,
     )
+    function_extensions = [
+        FUNCTION_DATETRUNC.extend(
+            dialect=ClickHouseDialect.CLICKHOUSE,
+            notes=(
+                Note(
+                    Translatable(
+                        "The function with three arguments is only available for the sources "
+                        "{dialects:CLICKHOUSE_21_8} or higher."
+                    ),
+                ),
+            ),
+        ),
+        FUNCTION_DATE.extend(
+            dialect=ClickHouseDialect.CLICKHOUSE,
+            notes=(
+                Note(
+                    Translatable(
+                        "For {dialects:CLICKHOUSE} data sources, numeric {arg:0} values less than or "
+                        "equal to `65535` are interpreted as the number of days (not seconds, like in "
+                        "all other cases) since January 1st 1970. This is the result of the behavior "
+                        "of available {dialects:CLICKHOUSE} functions.\n"
+                        "\n"
+                        "One way to surpass this is to use the following formula: "
+                        "`DATE(DATETIME([value]))`. The result is more consistent, but is likely to "
+                        "be much slower."
+                    ),
+                    level=NoteLevel.warning,
+                ),
+            ),
+        ),
+        FUNCTION_OP_COMPARISON.extend(
+            dialect=ClickHouseDialect.CLICKHOUSE,
+            notes=(
+                Note(
+                    Translatable(
+                        "Due to implementation details of the {type:FLOAT} type in {dialects:CLICKHOUSE} sources "
+                        "it is recommended to use the {ref:COMPARE} function instead of comparison operators for this type."
+                    ),
+                ),
+            ),
+        ),
+    ]

lib/dl_connector_clickhouse/dl_connector_clickhouse/locales/en/LC_MESSAGES/dl_connector_clickhouse.po

@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2023-09-22 08:15+0000\n"
+"POT-Creation-Date: 2023-09-22 08:16+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

lib/dl_connector_clickhouse/dl_connector_clickhouse/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_clickhouse.po

@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2023-09-22 08:15+0000\n"
+"POT-Creation-Date: 2023-09-22 08:16+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -14,3 +14,25 @@ msgstr ""
 
 msgid "`ClickHouse` version `22.10`"
 msgstr ""
+
+msgid ""
+"The function with three arguments is only available for the sources "
+"{dialects:CLICKHOUSE_21_8} or higher."
+msgstr ""
+
+msgid ""
+"For {dialects:CLICKHOUSE} data sources, numeric {arg:0} values less than or "
+"equal to `65535` are interpreted as the number of days (not seconds, like in "
+"all other cases) since January 1st 1970. This is the result of the behavior "
+"of available {dialects:CLICKHOUSE} functions.\n"
+"\n"
+"One way to surpass this is to use the following formula: "
+"`DATE(DATETIME([value]))`. The result is more consistent, but is likely to "
+"be much slower."
+msgstr ""
+
+msgid ""
+"Due to implementation details of the {type:FLOAT} type in {dialects:"
+"CLICKHOUSE} sources it is recommended to use the {ref:COMPARE} function "
+"instead of comparison operators for this type."
+msgstr ""

lib/dl_connector_clickhouse/dl_connector_clickhouse/locales/ru/LC_MESSAGES/dl_connector_clickhouse.po

@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2023-09-22 08:15+0000\n"
+"POT-Creation-Date: 2023-09-22 08:16+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

lib/dl_connector_clickhouse/dl_connector_clickhouse/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_clickhouse.po

@@ -4,7 +4,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2023-09-22 08:15+0000\n"
+"POT-Creation-Date: 2023-09-22 08:16+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -14,3 +14,38 @@ msgstr "`ClickHouse` версии `21.8`"
 
 msgid "`ClickHouse` version `22.10`"
 msgstr "`ClickHouse` версии `22.10`"
+
+msgid ""
+"The function with three arguments is only available for the sources "
+"{dialects:CLICKHOUSE_21_8} or higher."
+msgstr ""
+"Функция с тремя аргументами доступна только для источников {dialects:"
+"CLICKHOUSE_21_8} и выше."
+
+msgid ""
+"For {dialects:CLICKHOUSE} data sources, numeric {arg:0} values less than or "
+"equal to `65535` are interpreted as the number of days (not seconds, like in "
+"all other cases) since January 1st 1970. This is the result of the behavior "
+"of available {dialects:CLICKHOUSE} functions.\n"
+"\n"
+"One way to surpass this is to use the following formula: "
+"`DATE(DATETIME([value]))`. The result is more consistent, but is likely to "
+"be much slower."
+msgstr ""
+"Источники {dialects:CLICKHOUSE} воспринимают числовые значения {arg:0} "
+"меньше или равные `65535` как количество дней (не секунд, как во всех "
+"остальных случаях) с 1 января 1970 г. Это следствие поведения доступных в "
+"{dialects:CLICKHOUSE} функций.\n"
+"\n"
+"Чтобы обойти это, воспользуйтесь формулой: `DATE(DATETIME([value]))`. "
+"Формула имеет более корректное поведение, но может работать значительно "
+"медленнее."
+
+msgid ""
+"Due to implementation details of the {type:FLOAT} type in {dialects:"
+"CLICKHOUSE} sources it is recommended to use the {ref:COMPARE} function "
+"instead of comparison operators for this type."
+msgstr ""
+"Из-за особенностей реализации типа {type:FLOAT} в источниках {dialects:"
+"CLICKHOUSE} рекомендуется для этого типа использовать функцию {ref:COMPARE} "
+"вместо операторов сравнения."

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py

@@ -1,11 +1,10 @@
 import pytest
 
-from dl_constants.enums import RawSQLLevel
-
 from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration
 from dl_api_lib_testing.connection_base import ConnectionTestBase
 from dl_api_lib_testing.data_api_base import StandardizedDataApiTestBase
 from dl_api_lib_testing.dataset_base import DatasetTestBase
+from dl_constants.enums import RawSQLLevel
 
 from dl_connector_clickhouse.core.clickhouse.constants import SOURCE_TYPE_CH_TABLE
 from dl_connector_clickhouse.core.clickhouse_base.constants import CONNECTION_TYPE_CLICKHOUSE

lib/dl_connector_metrica/dl_connector_metrica/formula_ref/i18n.py

@@ -0,0 +1,23 @@
+import os
+
+import attr
+
+from dl_i18n.localizer_base import Translatable as BaseTranslatable
+from dl_i18n.localizer_base import TranslationConfig
+
+import dl_connector_metrica as package
+
+
+DOMAIN = f"dl_formula_ref_{package.__name__}"
+
+_LOCALE_DIR = os.path.join(os.path.dirname(__file__), "..", "locales")
+
+CONFIGS = [
+    TranslationConfig(path=_LOCALE_DIR, domain=DOMAIN, locale="en"),
+    TranslationConfig(path=_LOCALE_DIR, domain=DOMAIN, locale="ru"),
+]
+
+
+@attr.s
+class Translatable(BaseTranslatable):
+    domain: str = attr.ib(default=DOMAIN)

lib/dl_connector_metrica/dl_connector_metrica/formula_ref/plugin.py

@@ -1,9 +1,30 @@
+from dl_formula_ref.functions.operator import FUNCTION_OP_PLUS
 from dl_formula_ref.plugins.base.plugin import FormulaRefPlugin
+from dl_formula_ref.registry.note import (
+    Note,
+    NoteLevel,
+)
 
 from dl_connector_metrica.formula.constants import MetricaDialect
 from dl_connector_metrica.formula_ref.human_dialects import HUMAN_DIALECTS
+from dl_connector_metrica.formula_ref.i18n import (
+    CONFIGS,
+    Translatable,
+)
 
 
 class MetricaFormulaRefPlugin(FormulaRefPlugin):
     any_dialects = frozenset((*MetricaDialect.METRIKAAPI.to_list(),))
     human_dialects = HUMAN_DIALECTS
+    translation_configs = frozenset(CONFIGS)
+    function_extensions = [
+        FUNCTION_OP_PLUS.extend(
+            dialect=MetricaDialect.METRIKAAPI,
+            notes=[
+                Note(
+                    level=NoteLevel.warning,
+                    text=Translatable("{dialects: METRIKAAPI} does not support string concatenation."),
+                )
+            ],
+        ),
+    ]

lib/dl_connector_metrica/dl_connector_metrica/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_metrica.po

@@ -0,0 +1,13 @@
+# Copyright (c) 2023 YANDEX LLC <[email protected]>
+# This file is distributed under the same license as the DataLens package.
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: [email protected]\n"
+"POT-Creation-Date: 2023-11-06 10:45+0000\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "{dialects: METRIKAAPI} does not support string concatenation."
+msgstr ""

lib/dl_connector_metrica/dl_connector_metrica/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_metrica.po

@@ -0,0 +1,13 @@
+# Copyright (c) 2023 YANDEX LLC <[email protected]>
+# This file is distributed under the same license as the DataLens package.
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: [email protected]\n"
+"POT-Creation-Date: 2023-11-06 10:45+0000\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "{dialects: METRIKAAPI} does not support string concatenation."
+msgstr "{dialects: METRIKAAPI} не поддерживает конкатенацию строк."

lib/dl_connector_metrica/pyproject.toml

@@ -76,3 +76,6 @@ dl_connector_metrica = [
     {path = "dl_connector_metrica/api"},
     {path = "dl_connector_metrica/core"},
 ]
+dl_formula_ref_dl_connector_metrica = [
+    {path = "dl_connector_metrica/formula_ref"},
+]

lib/dl_connector_postgresql/dl_connector_postgresql/formula_ref/plugin.py

@@ -1,5 +1,7 @@
+from dl_formula_ref.functions.array import FUNCTION_UNNEST
 from dl_formula_ref.functions.type_conversion import DbCastExtension
 from dl_formula_ref.plugins.base.plugin import FormulaRefPlugin
+from dl_formula_ref.registry.note import Note
 
 from dl_connector_postgresql.formula.constants import PostgreSQLDialect
 from dl_connector_postgresql.formula.definitions.functions_type import FuncDbCastPostgreSQLBase
@@ -28,3 +30,17 @@ class PostgresSQLFormulaRefPlugin(FormulaRefPlugin):
             (PostgreSQLDialect.NON_COMPENG_POSTGRESQL, "varchar"): Translatable("Alias for `character varying`"),
         },
     )
+    function_extensions = [
+        FUNCTION_UNNEST.extend(
+            dialect=PostgreSQLDialect.NON_COMPENG_POSTGRESQL,
+            notes=(
+                Note(
+                    Translatable(
+                        "{dialects: POSTGRESQL} doesn't allow filtering fields containing the UNNEST "
+                        "function. If the data source is {dialects: POSTGRESQL}, do not use such "
+                        "fields in selectors."
+                    )
+                ),
+            ),
+        ),
+    ]

lib/dl_connector_postgresql/dl_connector_postgresql/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_postgresql.po

@@ -26,3 +26,9 @@ msgstr ""
 
 msgid "Alias for `character varying`"
 msgstr ""
+
+msgid ""
+"{dialects: POSTGRESQL} doesn't allow filtering fields containing the UNNEST "
+"function. If the data source is {dialects: POSTGRESQL}, do not use such "
+"fields in selectors."
+msgstr ""

lib/dl_connector_postgresql/dl_connector_postgresql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_postgresql.po

@@ -26,3 +26,12 @@ msgstr "Алиас для `character`"
 
 msgid "Alias for `character varying`"
 msgstr "Алиас для `character varying`"
+
+msgid ""
+"{dialects: POSTGRESQL} doesn't allow filtering fields containing the UNNEST "
+"function. If the data source is {dialects: POSTGRESQL}, do not use such "
+"fields in selectors."
+msgstr ""
+"{dialects: POSTGRESQL} запрещает фильтрацию по полям, содержащим функцию "
+"`UNNEST`. Не используйте такие поля для создания селектора, если источник "
+"данных — {dialects: POSTGRESQL}."

lib/dl_formula/dl_formula/core/dialect.py

@@ -2,12 +2,15 @@
 
 from __future__ import annotations
 
+from functools import reduce
 import inspect
 from itertools import chain
 import logging
+from operator import ior
 import re
 from typing import (
     Any,
+    Collection,
     Generator,
     NamedTuple,
     Optional,
@@ -283,6 +286,19 @@ def register_dialect_namespace(dialect_ns: Type[DialectNamespace]) -> None:
             _NAMED_DIALECT_COMBOS[name] = combo
 
 
+def dialect_combo_is_supported(supported: DialectCombo | Collection[DialectCombo], current: DialectCombo) -> bool:
+    supported_dcombo: DialectCombo
+    if not isinstance(supported, DialectCombo):
+        supported_dcombo = reduce(ior, supported)  # bitwise "or" (|) of the collection's elements
+    else:
+        supported_dcombo = supported
+
+    if supported_dcombo & StandardDialect.ANY:
+        return True
+
+    return supported_dcombo & current == current
+
+
 def get_dialect_combos() -> list[DialectCombo]:
     return list(_NAMED_DIALECT_COMBOS.values())