Add yadocs updating

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/api_schema/source.py

@@ -12,4 +12,5 @@ class Meta(BaseFileSourceSchema.Meta):
 
     public_link = fields.String(dump_only=True)
     private_path = fields.String(dump_only=True)
+    sheet_id = fields.String(dump_only=True)
     first_line_is_header = fields.Boolean(dump_only=True)

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/connection_info.py

@@ -6,4 +6,4 @@
 
 
 class YaDocsFileS3ConnectionInfoProvider(ConnectionInfoProvider):
-    title_translatable = Translatable("label_connector-yadocs")
+    title_translatable = Translatable("label_connector-docs")

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/adapter.py

@@ -1,11 +1,11 @@
 import logging
 
 from dl_connector_bundle_chs3.chs3_base.core.adapter import BaseAsyncFileS3Adapter
-from dl_connector_bundle_chs3.chs3_yadocs.core.constants import CONNECTION_TYPE_YADOCS
+from dl_connector_bundle_chs3.chs3_yadocs.core.constants import CONNECTION_TYPE_DOCS
 
 
 LOGGER = logging.getLogger(__name__)
 
 
 class AsyncYaDocsFileS3Adapter(BaseAsyncFileS3Adapter):
-    conn_type = CONNECTION_TYPE_YADOCS
+    conn_type = CONNECTION_TYPE_DOCS

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/connector.py

@@ -10,8 +10,8 @@
 from dl_connector_bundle_chs3.chs3_yadocs.core.adapter import AsyncYaDocsFileS3Adapter
 from dl_connector_bundle_chs3.chs3_yadocs.core.connection_executors import YaDocsFileS3AsyncAdapterConnExecutor
 from dl_connector_bundle_chs3.chs3_yadocs.core.constants import (
-    CONNECTION_TYPE_YADOCS,
-    SOURCE_TYPE_YADOCS,
+    CONNECTION_TYPE_DOCS,
+    SOURCE_TYPE_DOCS,
 )
 from dl_connector_bundle_chs3.chs3_yadocs.core.data_source import YaDocsFileS3DataSource
 from dl_connector_bundle_chs3.chs3_yadocs.core.data_source_spec import YaDocsFileS3DataSourceSpec
@@ -26,7 +26,7 @@
 
 
 class YaDocsFileS3CoreConnectionDefinition(BaseFileS3CoreConnectionDefinition):
-    conn_type = CONNECTION_TYPE_YADOCS
+    conn_type = CONNECTION_TYPE_DOCS
     connection_cls = YaDocsFileS3Connection
     us_storage_schema_cls = YaDocsFileConnectionDataStorageSchema
     sync_conn_executor_cls = YaDocsFileS3AsyncAdapterConnExecutor
@@ -37,7 +37,7 @@ class YaDocsFileS3CoreConnectionDefinition(BaseFileS3CoreConnectionDefinition):
 
 
 class YaDocsFileS3TableCoreSourceDefinition(BaseFileS3TableCoreSourceDefinition):
-    source_type = SOURCE_TYPE_YADOCS
+    source_type = SOURCE_TYPE_DOCS
     source_cls = YaDocsFileS3DataSource
     source_spec_cls = YaDocsFileS3DataSourceSpec
     us_storage_schema_cls = YaDocsFileS3DataSourceSpecStorageSchema

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/constants.py

@@ -4,5 +4,5 @@
 )
 
 
-CONNECTION_TYPE_YADOCS = ConnectionType.declare("docs")
-SOURCE_TYPE_YADOCS = DataSourceType.declare("YADOCS")
+CONNECTION_TYPE_DOCS = ConnectionType.declare("docs")
+SOURCE_TYPE_DOCS = DataSourceType.declare("DOCS")

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/data_source.py

@@ -10,18 +10,18 @@
 from dl_connector_bundle_chs3.chs3_base.core.constants import NOTIF_TYPE_DATA_UPDATE_FAILURE
 from dl_connector_bundle_chs3.chs3_base.core.data_source import BaseFileS3DataSource
 from dl_connector_bundle_chs3.chs3_yadocs.core.constants import (
-    CONNECTION_TYPE_YADOCS,
-    SOURCE_TYPE_YADOCS,
+    CONNECTION_TYPE_DOCS,
+    SOURCE_TYPE_DOCS,
 )
 
 
 class YaDocsFileS3DataSource(BaseFileS3DataSource):
-    conn_type = CONNECTION_TYPE_YADOCS
+    conn_type = CONNECTION_TYPE_DOCS
 
     @classmethod
     def is_compatible_with_type(cls, source_type: DataSourceType) -> bool:
         return source_type in {
-            SOURCE_TYPE_YADOCS,
+            SOURCE_TYPE_DOCS,
         }
 
     def _handle_component_errors(self) -> None:

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/settings.py

@@ -23,7 +23,7 @@ def yadocs_file_s3_settings_fallback(full_cfg: ConnectorsConfigType) -> dict[str
     if cfg is None:
         return {}
     return dict(
-        YADOCS=FileS3ConnectorSettings(  # type: ignore
+        DOCS=FileS3ConnectorSettings(  # type: ignore
             HOST=cfg.CONN_FILE_CH_HOST,
             PORT=cfg.CONN_FILE_CH_PORT,
             USERNAME=cfg.CONN_FILE_CH_USERNAME,

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/testing/connection.py

@@ -5,7 +5,7 @@
 
 from dl_core.us_manager.us_manager_sync import SyncUSManager
 
-from dl_connector_bundle_chs3.chs3_yadocs.core.constants import CONNECTION_TYPE_YADOCS
+from dl_connector_bundle_chs3.chs3_yadocs.core.constants import CONNECTION_TYPE_DOCS
 from dl_connector_bundle_chs3.chs3_yadocs.core.us_connection import YaDocsFileS3Connection
 
 
@@ -14,15 +14,15 @@ def make_saved_yadocs_connection(
     sources: list[YaDocsFileS3Connection.FileDataSource],
     **kwargs: Any,
 ) -> YaDocsFileS3Connection:
-    conn_type = CONNECTION_TYPE_YADOCS
+    conn_type = CONNECTION_TYPE_DOCS
 
     conn_name = "{} test conn {}".format(conn_type.name, uuid.uuid4())
     conn = YaDocsFileS3Connection.create_from_dict(
         data_dict=YaDocsFileS3Connection.DataModel(
             sources=sources,
         ),
         ds_key=conn_name,
-        type_=CONNECTION_TYPE_YADOCS.name,
+        type_=CONNECTION_TYPE_DOCS.name,
         us_manager=sync_usm,
         **kwargs,
     )

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/core/us_connection.py

@@ -17,21 +17,22 @@
 from dl_utils.utils import DataKey
 
 from dl_connector_bundle_chs3.chs3_base.core.us_connection import BaseFileS3Connection
-from dl_connector_bundle_chs3.chs3_yadocs.core.constants import SOURCE_TYPE_YADOCS
+from dl_connector_bundle_chs3.chs3_yadocs.core.constants import SOURCE_TYPE_DOCS
 
 
 LOGGER = logging.getLogger(__name__)
 
 
 class YaDocsFileS3Connection(BaseFileS3Connection):
-    source_type = SOURCE_TYPE_YADOCS
-    allowed_source_types = frozenset((SOURCE_TYPE_YADOCS,))
+    source_type = SOURCE_TYPE_DOCS
+    allowed_source_types = frozenset((SOURCE_TYPE_DOCS,))
 
     editable_data_source_parameters: ClassVar[
         tuple[str, ...]
     ] = BaseFileS3Connection.editable_data_source_parameters + (
         "public_link",
         "private_path",
+        "sheet_id",
         "first_line_is_header",
         "data_updated_at",
     )
@@ -40,6 +41,7 @@ class YaDocsFileS3Connection(BaseFileS3Connection):
     class FileDataSource(BaseFileS3Connection.FileDataSource):
         public_link: Optional[str] = attr.ib(default=None)
         private_path: Optional[str] = attr.ib(default=None)
+        sheet_id: Optional[str] = attr.ib(default=None)
         first_line_is_header: Optional[bool] = attr.ib(default=None)
         data_updated_at: datetime.datetime = attr.ib(factory=lambda: datetime.datetime.now(datetime.timezone.utc))
 
@@ -54,6 +56,7 @@ def __hash__(self) -> int:
                     self.s3_filename_suffix,
                     raw_schema,
                     self.status,
+                    self.sheet_id,
                     self.public_link,
                     self.private_path,
                 )
@@ -65,6 +68,7 @@ def str_for_hash(self) -> str:
                     super().str_for_hash(),
                     str(self.public_link),
                     str(self.private_path),
+                    str(self.sheet_id),
                 ]
             )
 
@@ -77,6 +81,7 @@ def get_desc(self) -> YaDocsFileSourceDesc:
                 preview_id=self.preview_id,
                 public_link=self.public_link,
                 private_path=self.private_path,
+                sheet_id=self.sheet_id,
                 first_line_is_header=self.first_line_is_header,
             )
 
@@ -124,6 +129,7 @@ def restore_source_params_from_orig(self, src_id: str, original_version: BaseFil
             first_line_is_header=orig_src.first_line_is_header,
             public_link=orig_src.public_link,
             private_path=orig_src.private_path,
+            sheet_id=orig_src.sheet_id,
         )
 
     @property

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/locales/en/LC_MESSAGES/dl_connector_bundle_chs3.po

@@ -12,7 +12,7 @@ msgstr ""
 msgid "label_connector-gsheets_v2"
 msgstr "Google Sheets"
 
-msgid "label_connector-yadocs"
+msgid "label_connector-docs"
 msgstr "Yandex Documents"
 
 msgid "label_connector-file"

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/locales/ru/LC_MESSAGES/dl_connector_bundle_chs3.po

@@ -12,7 +12,7 @@ msgstr ""
 msgid "label_connector-gsheets_v2"
 msgstr "Google Sheets"
 
-msgid "label_connector-yadocs"
+msgid "label_connector-docs"
 msgstr "Яндекс Документы"
 
 msgid "label_connector-file"

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py

@@ -11,7 +11,7 @@
     host_us_pg=get_test_container_hostport("pg-us", fallback_port=52610).host,
     port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=52610).port,
     us_master_token="AC1ofiek8coB",
-    core_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"],
+    core_connector_ep_names=["clickhouse", "file", "gsheets_v2", "docs"],
     redis_host=get_test_container_hostport("redis", fallback_port=52604).host,
     redis_port=get_test_container_hostport("redis", fallback_port=52604).port,
     redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5",
@@ -37,7 +37,7 @@
 S3_ENDPOINT_URL = f"http://{get_test_container_hostport('s3-storage', fallback_port=52620).as_pair()}"
 
 API_TEST_CONFIG = ApiTestEnvironmentConfiguration(
-    api_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"],
+    api_connector_ep_names=["clickhouse", "file", "gsheets_v2", "docs"],
     core_test_config=CORE_TEST_CONFIG,
     ext_query_executer_secret_key="_some_test_secret_key_",
 )

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/api/base.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/api/base.py

@@ -11,7 +11,7 @@
 from dl_connector_bundle_chs3_tests.db.base.api.base import CHS3ConnectionApiTestBase
 from dl_connector_bundle_chs3_tests.db.base.api.data import CHS3DataApiTestBase
 from dl_connector_bundle_chs3_tests.db.base.api.dataset import CHS3DatasetTestBase
-from dl_connector_bundle_chs3_tests.db.yadocs.core.base import BaseYaDocsFileS3TestClass
+from dl_connector_bundle_chs3_tests.db.docs.core.base import BaseYaDocsFileS3TestClass
 
 
 LOGGER = logging.getLogger(__name__)

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/api/test_connection.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/api/test_connection.py

@@ -10,7 +10,7 @@
 
 from dl_connector_bundle_chs3.chs3_yadocs.core.us_connection import YaDocsFileS3Connection
 from dl_connector_bundle_chs3_tests.db.base.api.connection import CHS3ConnectionTestSuite
-from dl_connector_bundle_chs3_tests.db.yadocs.api.base import YaDocsFileS3ApiConnectionTestBase
+from dl_connector_bundle_chs3_tests.db.docs.api.base import YaDocsFileS3ApiConnectionTestBase
 
 
 class TestYaDocsFileS3Connection(YaDocsFileS3ApiConnectionTestBase, CHS3ConnectionTestSuite[YaDocsFileS3Connection]):

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/api/test_data.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/api/test_data.py

@@ -27,7 +27,7 @@
 from dl_connector_bundle_chs3.chs3_yadocs.core.lifecycle import YaDocsFileS3ConnectionLifecycleManager
 from dl_connector_bundle_chs3.chs3_yadocs.core.us_connection import YaDocsFileS3Connection
 from dl_connector_bundle_chs3_tests.db.base.api.data import CHS3DataResultTestSuite
-from dl_connector_bundle_chs3_tests.db.yadocs.api.base import YaDocsFileS3DataApiTestBase
+from dl_connector_bundle_chs3_tests.db.docs.api.base import YaDocsFileS3DataApiTestBase
 
 
 class TestYaDocsFileS3DataResult(YaDocsFileS3DataApiTestBase, CHS3DataResultTestSuite):

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/api/test_dataset.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/api/test_dataset.py

@@ -1,5 +1,5 @@
 from dl_connector_bundle_chs3_tests.db.base.api.dataset import CHS3DatasetTestSuite
-from dl_connector_bundle_chs3_tests.db.yadocs.api.base import YaDocsFileS3DatasetTestBase
+from dl_connector_bundle_chs3_tests.db.docs.api.base import YaDocsFileS3DatasetTestBase
 
 
 class TestYaDocsFileS3Dataset(YaDocsFileS3DatasetTestBase, CHS3DatasetTestSuite):

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/core/base.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/core/base.py

@@ -7,17 +7,17 @@
 from dl_core_testing.fixtures.primitives import FixtureTableSpec
 
 from dl_connector_bundle_chs3.chs3_yadocs.core.constants import (
-    CONNECTION_TYPE_YADOCS,
-    SOURCE_TYPE_YADOCS,
+    CONNECTION_TYPE_DOCS,
+    SOURCE_TYPE_DOCS,
 )
 from dl_connector_bundle_chs3.chs3_yadocs.core.testing.connection import make_saved_yadocs_connection
 from dl_connector_bundle_chs3.chs3_yadocs.core.us_connection import YaDocsFileS3Connection
 from dl_connector_bundle_chs3_tests.db.base.core.base import BaseCHS3TestClass
 
 
 class BaseYaDocsFileS3TestClass(BaseCHS3TestClass[YaDocsFileS3Connection]):
-    conn_type = CONNECTION_TYPE_YADOCS
-    source_type = SOURCE_TYPE_YADOCS
+    conn_type = CONNECTION_TYPE_DOCS
+    source_type = SOURCE_TYPE_DOCS
 
     @pytest.fixture(scope="function")
     def sample_file_data_source(

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/core/test_adapter.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/core/test_adapter.py

@@ -2,7 +2,7 @@
 
 from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO
 from dl_connector_bundle_chs3.chs3_yadocs.core.adapter import AsyncYaDocsFileS3Adapter
-from dl_connector_bundle_chs3_tests.db.yadocs.core.base import BaseYaDocsFileS3TestClass
+from dl_connector_bundle_chs3_tests.db.docs.core.base import BaseYaDocsFileS3TestClass
 
 
 class TestAsyncYaDocsFileS3Adapter(

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/core/test_connection.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/core/test_connection.py

@@ -2,7 +2,7 @@
 
 from dl_connector_bundle_chs3.file.core.us_connection import FileS3Connection
 from dl_connector_bundle_chs3_tests.db.base.core.connection import CHS3ConnectionTestBase
-from dl_connector_bundle_chs3_tests.db.yadocs.core.base import BaseYaDocsFileS3TestClass
+from dl_connector_bundle_chs3_tests.db.docs.core.base import BaseYaDocsFileS3TestClass
 
 
 class TestYaDocsFileS3Connection(BaseYaDocsFileS3TestClass, CHS3ConnectionTestBase[FileS3Connection]):

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/core/test_connection_executor.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/core/test_connection_executor.py

@@ -3,7 +3,7 @@
     CHS3AsyncConnectionExecutorTestBase,
     CHS3SyncConnectionExecutorTestBase,
 )
-from dl_connector_bundle_chs3_tests.db.yadocs.core.base import BaseYaDocsFileS3TestClass
+from dl_connector_bundle_chs3_tests.db.docs.core.base import BaseYaDocsFileS3TestClass
 
 
 class TestYaDocsFileS3SyncConnectionExecutor(

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/core/test_data_source.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/core/test_data_source.py

@@ -2,7 +2,7 @@
 from dl_connector_bundle_chs3.chs3_yadocs.core.data_source_spec import YaDocsFileS3DataSourceSpec
 from dl_connector_bundle_chs3.chs3_yadocs.core.us_connection import YaDocsFileS3Connection
 from dl_connector_bundle_chs3_tests.db.base.core.data_source import CHS3TableDataSourceTestBase
-from dl_connector_bundle_chs3_tests.db.yadocs.core.base import BaseYaDocsFileS3TestClass
+from dl_connector_bundle_chs3_tests.db.docs.core.base import BaseYaDocsFileS3TestClass
 
 
 class TestYaDocsFileS3TableDataSource(

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/yadocs/core/test_dataset.py → lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/docs/core/test_dataset.py

@@ -1,6 +1,6 @@
 from dl_connector_bundle_chs3.chs3_yadocs.core.us_connection import YaDocsFileS3Connection
 from dl_connector_bundle_chs3_tests.db.base.core.dataset import CHS3DatasetTestBase
-from dl_connector_bundle_chs3_tests.db.yadocs.core.base import BaseYaDocsFileS3TestClass
+from dl_connector_bundle_chs3_tests.db.docs.core.base import BaseYaDocsFileS3TestClass
 
 
 class TestYaDocsFileS3Dataset(BaseYaDocsFileS3TestClass, CHS3DatasetTestBase[YaDocsFileS3Connection]):

lib/dl_connector_bundle_chs3/pyproject.toml

@@ -34,12 +34,12 @@ datalens-task-processor = {path = "../dl_task_processor"}
 [tool.poetry.plugins."dl_api_lib.connectors"]
 file = "dl_connector_bundle_chs3.file.api.connector:FileS3ApiConnector"
 gsheets_v2 = "dl_connector_bundle_chs3.chs3_gsheets.api.connector:GSheetsFileS3ApiConnector"
-yadocs = "dl_connector_bundle_chs3.chs3_yadocs.api.connector:YaDocsFileS3ApiConnector"
+docs = "dl_connector_bundle_chs3.chs3_yadocs.api.connector:YaDocsFileS3ApiConnector"
 
 [tool.poetry.plugins."dl_core.connectors"]
 file = "dl_connector_bundle_chs3.file.core.connector:FileS3CoreConnector"
 gsheets_v2 = "dl_connector_bundle_chs3.chs3_gsheets.core.connector:GSheetsFileS3CoreConnector"
-yadocs = "dl_connector_bundle_chs3.chs3_yadocs.core.connector:YaDocsFileS3CoreConnector"
+docs = "dl_connector_bundle_chs3.chs3_yadocs.core.connector:YaDocsFileS3CoreConnector"
 
 [tool.poetry.group.tests.dependencies]
 pytest = ">=7.2.2"
@@ -63,9 +63,9 @@ target_path = "file"
 root_dir = "dl_connector_bundle_chs3_tests/db"
 target_path = "gsheets_v2"
 
-[datalens.pytest.db_yadocs]
+[datalens.pytest.db_docs]
 root_dir = "dl_connector_bundle_chs3_tests/db"
-target_path = "yadocs"
+target_path = "docs"
 
 [tool.mypy]
 warn_unused_configs = true

lib/dl_core/dl_core/services_registry/file_uploader_client_factory.py

@@ -54,6 +54,7 @@ class YaDocsFileSourceDesc(FileSourceDesc):
     public_link: Optional[str] = attr.ib()
     private_path: Optional[str] = attr.ib()
     first_line_is_header: Optional[bool] = attr.ib()
+    sheet_id: Optional[str] = attr.ib()
 
 
 @attr.s(frozen=True)

lib/dl_file_uploader_api_lib/dl_file_uploader_api_lib/app.py

@@ -62,7 +62,7 @@ def set_up_sentry(self, secret_sentry_dsn: str, release: str) -> None:
         )
 
     def create_app(self, app_version: str) -> web.Application:
-        core_conn_whitelist = ["clickhouse", "file", "gsheets_v2"]
+        core_conn_whitelist = ["clickhouse", "file", "gsheets_v2", "docs"]
         load_core_lib(core_lib_config=CoreLibraryConfig(core_connector_ep_names=core_conn_whitelist))
 
         if (secret_sentry_dsn := self._settings.SENTRY_DSN) is not None: