diff --git a/lib/dl_api_lib/dl_api_lib/api_common/dataset_loader.py b/lib/dl_api_lib/dl_api_lib/api_common/dataset_loader.py index 4401b3173..a60e61028 100644 --- a/lib/dl_api_lib/dl_api_lib/api_common/dataset_loader.py +++ b/lib/dl_api_lib/dl_api_lib/api_common/dataset_loader.py @@ -13,7 +13,6 @@ from dl_api_lib import exc from dl_api_lib.dataset.utils import allow_rls_for_dataset from dl_api_lib.service_registry.service_registry import ApiServiceRegistry -from dl_api_lib.utils.rls import FieldRLSSerializer from dl_app_tools.profiling_base import generic_profiler from dl_constants.exc import ( DEFAULT_ERR_CODE_API_PREFIX, @@ -36,11 +35,13 @@ from dl_core.us_manager.local_cache import USEntryBuffer from dl_core.us_manager.us_manager import USManagerBase from dl_core.us_manager.us_manager_sync import SyncUSManager +import dl_rls.exc as rls_exc +from dl_rls.serializer import FieldRLSSerializer from dl_utils.aio import await_sync if TYPE_CHECKING: - from dl_core.rls import RLSEntry + from dl_rls.models import RLSEntry LOGGER = logging.getLogger(__name__) @@ -305,7 +306,7 @@ def _update_dataset_rls_from_body(self, dataset: Dataset, body: dict, allow_rls_ if rlse.field_guid == field.guid ] if self._rls_list_to_set(saved_field_rls) != self._rls_list_to_set(rls_entries_pre): - raise exc.RLSConfigParsingError( + raise rls_exc.RLSConfigParsingError( "For this feature to work, save dataset after editing the RLS config.", details=dict() ) # otherwise no effective config changes (that are worth checking in preview) diff --git a/lib/dl_api_lib/dl_api_lib/app/control_api/resources/dataset_base.py b/lib/dl_api_lib/dl_api_lib/app/control_api/resources/dataset_base.py index 2e1c262d3..16a4bead6 100644 --- a/lib/dl_api_lib/dl_api_lib/app/control_api/resources/dataset_base.py +++ b/lib/dl_api_lib/dl_api_lib/app/control_api/resources/dataset_base.py @@ -29,7 +29,6 @@ is_compeng_executable, ) from dl_api_lib.service_registry.service_registry import ApiServiceRegistry -from dl_api_lib.utils.rls import FieldRLSSerializer from dl_constants.enums import ( AggregationFunction, BinaryJoinOperator, @@ -61,6 +60,7 @@ StandardDialect, from_name_and_version, ) +from dl_rls.serializer import FieldRLSSerializer LOGGER = logging.getLogger(__name__) diff --git a/lib/dl_api_lib/dl_api_lib/app_common.py b/lib/dl_api_lib/dl_api_lib/app_common.py index 63f2507ff..0406ba473 100644 --- a/lib/dl_api_lib/dl_api_lib/app_common.py +++ b/lib/dl_api_lib/dl_api_lib/app_common.py @@ -28,11 +28,6 @@ ConnectionType, RLSSubjectType, ) -from dl_core.rls import ( - RLS_FAILED_USER_NAME_PREFIX, - BaseSubjectResolver, - RLSSubject, -) from dl_core.services_registry.entity_checker import EntityUsageChecker from dl_core.services_registry.file_uploader_client_factory import FileUploaderSettings from dl_core.services_registry.inst_specific_sr import ( @@ -48,6 +43,11 @@ ) from dl_pivot.base.transformer_factory import PivotTransformerFactory from dl_pivot.plugin_registration import get_pivot_transformer_factory_cls +from dl_rls.models import ( + RLS_FAILED_USER_NAME_PREFIX, + RLSSubject, +) +from dl_rls.subject_resolver import BaseSubjectResolver from dl_task_processor.arq_wrapper import create_arq_redis_settings diff --git a/lib/dl_api_lib/dl_api_lib/error_handling.py b/lib/dl_api_lib/dl_api_lib/error_handling.py index 94f133989..d958c5523 100644 --- a/lib/dl_api_lib/dl_api_lib/error_handling.py +++ b/lib/dl_api_lib/dl_api_lib/error_handling.py @@ -26,6 +26,7 @@ from dl_dashsql import exc as dashsql_exc from dl_formula.core import exc as formula_exc import dl_query_processing.exc +from dl_rls import exc as rls_exc LOGGER = logging.getLogger(__name__) @@ -56,8 +57,7 @@ common_exc.InvalidFieldError: status.BAD_REQUEST, common_exc.FieldNotFound: status.BAD_REQUEST, common_exc.USPermissionRequired: status.FORBIDDEN, - exc.RLSConfigParsingError: status.BAD_REQUEST, - common_exc.RLSSubjectNotFound: status.BAD_REQUEST, + rls_exc.RLSError: status.BAD_REQUEST, exc.FeatureNotAvailable: status.BAD_REQUEST, dl_query_processing.exc.FilterError: status.BAD_REQUEST, exc.UnsupportedForEntityType: status.BAD_REQUEST, diff --git a/lib/dl_api_lib/dl_api_lib/exc.py b/lib/dl_api_lib/dl_api_lib/exc.py index 74489bac1..83d180bb5 100644 --- a/lib/dl_api_lib/dl_api_lib/exc.py +++ b/lib/dl_api_lib/dl_api_lib/exc.py @@ -7,14 +7,6 @@ class FeatureNotAvailable(DLBaseException): err_code = DLBaseException.err_code + ["FEATURE_NOT_AVAILABLE"] -class RLSError(DLBaseException): - err_code = DLBaseException.err_code + ["RLS"] - - -class RLSConfigParsingError(RLSError): - err_code = RLSError.err_code + ["PARSE"] - - class DatasetActionNotAllowedError(DLBaseException): err_code = DLBaseException.err_code + ["ACTION_NOT_ALLOWED"] diff --git a/lib/dl_api_lib/dl_api_lib/service_registry/service_registry.py b/lib/dl_api_lib/dl_api_lib/service_registry/service_registry.py index 3e0d32740..34d069558 100644 --- a/lib/dl_api_lib/dl_api_lib/service_registry/service_registry.py +++ b/lib/dl_api_lib/dl_api_lib/service_registry/service_registry.py @@ -20,7 +20,6 @@ DefaultQueryProcessorFactory, TypedQueryProcessorFactory, ) -from dl_api_lib.utils.rls import BaseSubjectResolver from dl_constants.enums import QueryProcessingMode from dl_core.services_registry.top_level import ( DefaultServicesRegistry, @@ -33,6 +32,7 @@ LocalizerFactory, ) from dl_pivot.base.transformer_factory import PivotTransformerFactory +from dl_rls.subject_resolver import BaseSubjectResolver if TYPE_CHECKING: diff --git a/lib/dl_api_lib/dl_api_lib/utils/__init__.py b/lib/dl_api_lib/dl_api_lib/utils/__init__.py index 447daa1f0..da9846c52 100644 --- a/lib/dl_api_lib/dl_api_lib/utils/__init__.py +++ b/lib/dl_api_lib/dl_api_lib/utils/__init__.py @@ -8,8 +8,6 @@ from __future__ import annotations from .base import ( # ... - chunks, - need_permission, need_permission_on_entry, profile_stats, query_execution_context, @@ -19,7 +17,5 @@ __all__ = ( "query_execution_context", "profile_stats", - "need_permission", "need_permission_on_entry", - "chunks", ) diff --git a/lib/dl_api_lib/dl_api_lib/utils/base.py b/lib/dl_api_lib/dl_api_lib/utils/base.py index 0a46b6e8b..26df1d6c8 100644 --- a/lib/dl_api_lib/dl_api_lib/utils/base.py +++ b/lib/dl_api_lib/dl_api_lib/utils/base.py @@ -9,20 +9,13 @@ import os from typing import ( TYPE_CHECKING, - Any, - Iterable, Iterator, Optional, ) import uuid from dl_api_lib.enums import USPermissionKind -from dl_app_tools.profiling_base import GenericProfiler import dl_core.exc as common_exc -from dl_core.flask_utils.us_manager_middleware import USManagerFlaskMiddleware - - -# noinspection PyUnresolvedReferences if TYPE_CHECKING: @@ -73,66 +66,8 @@ def profile_stats(stats_dir: Optional[str] = None) -> Iterator[None]: pr.dump_stats(filename) -def need_permission(us_entry_id: str, permission: USPermissionKind) -> None: - usm_user = USManagerFlaskMiddleware.get_request_us_manager() - entry = usm_user.get_by_id(us_entry_id) - need_permission_on_entry(entry, permission) - - def need_permission_on_entry(us_entry: USEntry, permission: USPermissionKind) -> None: assert us_entry.permissions is not None assert us_entry.uuid is not None if not us_entry.permissions[permission.name]: raise common_exc.USPermissionRequired(us_entry.uuid, permission.name) - - -def chunks(lst: list[Any], size: int) -> Iterable[list[Any]]: - """Yield successive chunks from lst. No padding.""" - for idx in range(0, len(lst), size): - yield lst[idx : idx + size] - - -def split_by_quoted_quote(value: str, quote: str = "'") -> tuple[str, str]: - """ - Parse out a quoted value at the beginning, - where quotes are quoted by doubling (CSV-like). - - >>> split_by_quoted_quote("'abc'de") - ('abc', 'de') - >>> split_by_quoted_quote("'ab''c'''de") - ("ab'c'", 'de') - >>> split_by_quoted_quote("'ab''c'''") - ("ab'c'", '') - """ - ql = len(quote) - if not value.startswith(quote): - raise ValueError("Value does not start with quote") - value = value[ql:] - result = [] - while True: - try: - next_quote = value.index(quote) - except ValueError as e: - raise ValueError("Unclosed quote") from e - value_piece = value[:next_quote] - result.append(value_piece) - value = value[next_quote + ql :] - if value.startswith(quote): - result.append(quote) - value = value[ql:] - else: # some other text, or end-of-line. - break - - return "".join(result), value - - -def quote_by_quote(value: str, quote: str = "'") -> str: - """ - ... - - >>> quote_by_quote("a'b'") - "'a''b'''" - >>> split_by_quoted_quote(quote_by_quote("a'b'") + "and 'stuff'") - ("a'b'", "and 'stuff'") - """ - return "{}{}{}".format(quote, value.replace(quote, quote + quote), quote) diff --git a/lib/dl_api_lib/dl_api_lib_tests/unit/test_rls.py b/lib/dl_api_lib/dl_api_lib_tests/unit/test_rls.py index 99e943831..25a283926 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/unit/test_rls.py +++ b/lib/dl_api_lib/dl_api_lib_tests/unit/test_rls.py @@ -1,12 +1,12 @@ import pytest -from dl_api_lib.utils.rls import FieldRLSSerializer from dl_api_lib_testing.rls import ( RLS_CONFIG_CASES, config_to_comparable, ) from dl_constants.enums import RLSSubjectType -from dl_core.rls import RLSSubject +from dl_rls.models import RLSSubject +from dl_rls.serializer import FieldRLSSerializer def test_group_names_by_account_type(): diff --git a/lib/dl_api_lib/pyproject.toml b/lib/dl_api_lib/pyproject.toml index dc1fe8a98..a7e10efc6 100644 --- a/lib/dl_api_lib/pyproject.toml +++ b/lib/dl_api_lib/pyproject.toml @@ -16,7 +16,6 @@ flask-marshmallow = ">=1.1.0" flask-restx = ">=1.1.0" marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" -more-itertools = ">=9.1.0" python = ">=3.10, <3.13" werkzeug = ">=2.2.3" statcommons = {path = "../statcommons"} @@ -37,6 +36,7 @@ datalens-dashsql = {path = "../../lib/dl_dashsql"} datalens-pivot = {path = "../dl_pivot"} datalens-pivot-pandas = {path = "../dl_pivot_pandas"} datalens-cache-engine = {path = "../dl_cache_engine"} +datalens-rls = {path = "../dl_rls"} [tool.poetry.group.tests.dependencies] pytest = ">=7.2.2" diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py index 35ec1adcc..d5c09d66e 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py @@ -37,11 +37,6 @@ ) from dl_core.aio.middlewares.services_registry import services_registry_middleware from dl_core.aio.middlewares.us_manager import service_us_manager_middleware -from dl_core.rls import ( - RLS_FAILED_USER_NAME_PREFIX, - BaseSubjectResolver, - RLSSubject, -) from dl_core.services_registry import ServicesRegistry from dl_core.services_registry.entity_checker import EntityUsageChecker from dl_core.services_registry.env_manager_factory_base import EnvManagerFactory @@ -53,6 +48,11 @@ from dl_core.utils import FutureRef from dl_core_testing.app_test_workarounds import TestEnvManagerFactory from dl_core_testing.fixture_server_runner import WSGIRunner +from dl_rls.models import ( + RLS_FAILED_USER_NAME_PREFIX, + RLSSubject, +) +from dl_rls.subject_resolver import BaseSubjectResolver from dl_testing.utils import get_root_certificates diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py b/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py index 49e5af774..4d24d4a01 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py @@ -1,15 +1,15 @@ import json import pkgutil -from dl_api_lib.utils.rls import FieldRLSSerializer import dl_api_lib_testing.test_data from dl_constants.enums import RLSSubjectType -from dl_core.rls import ( - BaseSubjectResolver, +from dl_rls.models import ( RLSEntry, RLSPatternType, RLSSubject, ) +from dl_rls.serializer import FieldRLSSerializer +from dl_rls.subject_resolver import BaseSubjectResolver def load_rls_config(name: str) -> str: diff --git a/lib/dl_api_lib_testing/pyproject.toml b/lib/dl_api_lib_testing/pyproject.toml index 2446f4d74..57eeb3156 100644 --- a/lib/dl_api_lib_testing/pyproject.toml +++ b/lib/dl_api_lib_testing/pyproject.toml @@ -31,6 +31,7 @@ datalens-pivot-pandas = {path = "../dl_pivot_pandas"} datalens-query-processing = {path = "../dl_query_processing"} datalens-testing = {path = "../dl_testing"} datalens-cache-engine = {path = "../dl_cache_engine"} +datalens-rls = {path = "../dl_rls"} [build-system] build-backend = "poetry.core.masonry.api" diff --git a/lib/dl_core/dl_core/exc.py b/lib/dl_core/dl_core/exc.py index b7b366e0f..31a6d51ad 100644 --- a/lib/dl_core/dl_core/exc.py +++ b/lib/dl_core/dl_core/exc.py @@ -663,7 +663,3 @@ def message(self) -> str: class DataSourceMigrationImpossible(DLBaseException): err_code = DLBaseException.err_code + ["DSRC_MIGRATION_IMPOSSIBLE"] - - -class RLSSubjectNotFound(DLBaseException): - err_code = DLBaseException.err_code + ["RLS_SUBJECT_NOT_FOUND"] diff --git a/lib/dl_core/dl_core/services_registry/inst_specific_sr.py b/lib/dl_core/dl_core/services_registry/inst_specific_sr.py index c021ef1ed..e1bc3e7c1 100644 --- a/lib/dl_core/dl_core/services_registry/inst_specific_sr.py +++ b/lib/dl_core/dl_core/services_registry/inst_specific_sr.py @@ -3,8 +3,8 @@ import attr -from dl_core.rls import BaseSubjectResolver from dl_core.utils import FutureRef +from dl_rls.subject_resolver import BaseSubjectResolver if TYPE_CHECKING: diff --git a/lib/dl_core/dl_core/us_dataset.py b/lib/dl_core/dl_core/us_dataset.py index 645851f99..2c3cf8bb8 100644 --- a/lib/dl_core/dl_core/us_dataset.py +++ b/lib/dl_core/dl_core/us_dataset.py @@ -42,11 +42,11 @@ DirectCalculationSpec, ResultSchema, ) -from dl_core.rls import RLS from dl_core.us_entry import ( BaseAttrsDataModel, USEntry, ) +from dl_rls.rls import RLS if TYPE_CHECKING: diff --git a/lib/dl_core/dl_core/us_manager/storage_schemas/dataset.py b/lib/dl_core/dl_core/us_manager/storage_schemas/dataset.py index a95c8cebe..2331e8a97 100644 --- a/lib/dl_core/dl_core/us_manager/storage_schemas/dataset.py +++ b/lib/dl_core/dl_core/us_manager/storage_schemas/dataset.py @@ -30,7 +30,6 @@ WhereClauseOperation, ) from dl_core import multisource -from dl_core import rls as rls_module from dl_core.base_models import ( DefaultWhereClause, ObligatoryFilter, @@ -75,6 +74,11 @@ TreeStrValue, UuidValue, ) +from dl_rls.models import ( + RLSEntry, + RLSSubject, +) +from dl_rls.rls import RLS class SourceAvatarSchema(DefaultStorageSchema): @@ -145,13 +149,13 @@ class JoinConditionSchema(DefaultStorageSchema): class RLSSchema(DefaultStorageSchema): - TARGET_CLS = rls_module.RLS + TARGET_CLS = RLS class RLSEntrySchema(DefaultStorageSchema): - TARGET_CLS = rls_module.RLSEntry + TARGET_CLS = RLSEntry class RLSSubjectSchema(DefaultStorageSchema): - TARGET_CLS = rls_module.RLSSubject + TARGET_CLS = RLSSubject subject_type = ma_fields.Enum(RLSSubjectType) subject_id = ma_fields.String() diff --git a/lib/dl_core/dl_core_tests/db/components/test_rls.py b/lib/dl_core/dl_core_tests/db/components/test_rls.py index 66ed9b6af..6315a3481 100644 --- a/lib/dl_core/dl_core_tests/db/components/test_rls.py +++ b/lib/dl_core/dl_core_tests/db/components/test_rls.py @@ -6,11 +6,11 @@ RLSSubjectType, ) from dl_core.fields import BIField -from dl_core.rls import ( +from dl_core_tests.db.base import DefaultCoreTestClass +from dl_rls.models import ( RLSEntry, RLSSubject, ) -from dl_core_tests.db.base import DefaultCoreTestClass class TestRLS(DefaultCoreTestClass): diff --git a/lib/dl_core/pyproject.toml b/lib/dl_core/pyproject.toml index a908bbb57..a253d69ff 100644 --- a/lib/dl_core/pyproject.toml +++ b/lib/dl_core/pyproject.toml @@ -52,6 +52,7 @@ datalens-model-tools = {path = "../dl_model_tools"} datalens-task-processor = {path = "../dl_task_processor"} datalens-dashsql = {path = "../../lib/dl_dashsql"} datalens-cache-engine = {path = "../dl_cache_engine"} +datalens-rls = {path = "../dl_rls"} [tool.poetry.group.tests.dependencies] flaky = "==3.8.1" diff --git a/lib/dl_rls/LICENSE b/lib/dl_rls/LICENSE new file mode 100644 index 000000000..74ba5f6c7 --- /dev/null +++ b/lib/dl_rls/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023 YANDEX LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/dl_rls/README.md b/lib/dl_rls/README.md new file mode 100644 index 000000000..77e472df8 --- /dev/null +++ b/lib/dl_rls/README.md @@ -0,0 +1,4 @@ +# dl_rls + +A dummy package used for initialization of new library packages +either manually or via `dl_repmanager` diff --git a/lib/dl_rls/dl_rls/__init__.py b/lib/dl_rls/dl_rls/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_rls/dl_rls/exc.py b/lib/dl_rls/dl_rls/exc.py new file mode 100644 index 000000000..9319b80f3 --- /dev/null +++ b/lib/dl_rls/dl_rls/exc.py @@ -0,0 +1,13 @@ +from dl_constants.exc import DLBaseException + + +class RLSError(DLBaseException): + err_code = DLBaseException.err_code + ["RLS"] + + +class RLSConfigParsingError(RLSError): + err_code = RLSError.err_code + ["PARSE"] + + +class RLSSubjectNotFound(RLSError): + err_code = DLBaseException.err_code + ["RLS_SUBJECT_NOT_FOUND"] diff --git a/lib/dl_rls/dl_rls/models.py b/lib/dl_rls/dl_rls/models.py new file mode 100644 index 000000000..47b622c4c --- /dev/null +++ b/lib/dl_rls/dl_rls/models.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Optional + +import attr + +from dl_constants.enums import ( + RLSPatternType, + RLSSubjectType, +) + + +@attr.s(slots=True) +class RLSSubject: + subject_type: RLSSubjectType = attr.ib() + subject_id: str = attr.ib() + subject_name: str = attr.ib() # login, group name, etc + + +@attr.s(slots=True) +class RLSEntry: + field_guid: str = attr.ib() + allowed_value: Optional[str] = attr.ib() + subject: RLSSubject = attr.ib() + # Note: this is a bit of a hack to avoid very extensive splitting of the + # RLSEntry into multiple classes. + # For `pattern_type=RLSPatternType.all`, `allowed_value` must be `None`. + # For `pattern_type=RLSPatternType.userid`, `allowed_value` must be `None`, + # and `subject` must be `RLSSubjectType.userid`. + pattern_type: RLSPatternType = attr.ib(default=RLSPatternType.value) + + def ensure_removed_failed_subject_prefix(self) -> RLSEntry: + rls_entry = deepcopy(self) + rls_entry.subject.subject_name = rls_entry.subject.subject_name.removeprefix(RLS_FAILED_USER_NAME_PREFIX) + return rls_entry + + +# Special type subject that denotes 'all subjects'. +RLS_ALL_SUBJECT_NAME = "*" +RLS_ALL_SUBJECT = RLSSubject( + subject_type=RLSSubjectType.all, subject_id=RLS_ALL_SUBJECT_NAME, subject_name=RLS_ALL_SUBJECT_NAME +) +RLS_FAILED_USER_NAME_PREFIX = "!FAILED_" +# Special type subject that denotes 'insert userid'. +RLS_USERID_SUBJECT_NAME = "userid" +RLS_USERID_SUBJECT = RLSSubject(subject_type=RLSSubjectType.userid, subject_id="", subject_name=RLS_USERID_SUBJECT_NAME) diff --git a/lib/dl_rls/dl_rls/py.typed b/lib/dl_rls/dl_rls/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_core/dl_core/rls.py b/lib/dl_rls/dl_rls/rls.py similarity index 54% rename from lib/dl_core/dl_core/rls.py rename to lib/dl_rls/dl_rls/rls.py index 206d19906..ccd08d8f5 100644 --- a/lib/dl_core/dl_core/rls.py +++ b/lib/dl_rls/dl_rls/rls.py @@ -6,13 +6,10 @@ from __future__ import annotations -import abc -import copy from typing import ( - Dict, - List, + NamedTuple, Optional, - Tuple, + cast, ) import attr @@ -21,60 +18,30 @@ RLSPatternType, RLSSubjectType, ) -from dl_utils.utils import split_list +from dl_rls.models import RLSEntry -@attr.s(slots=True) -class RLSSubject: - subject_type: RLSSubjectType = attr.ib() - subject_id: str = attr.ib() - subject_name: str = attr.ib() # login, group name, etc - - -# Special type subject that denotes 'all subjects'. -RLS_ALL_SUBJECT_NAME = "*" -RLS_ALL_SUBJECT = RLSSubject( - subject_type=RLSSubjectType.all, subject_id=RLS_ALL_SUBJECT_NAME, subject_name=RLS_ALL_SUBJECT_NAME -) -RLS_FAILED_USER_NAME_PREFIX = "!FAILED_" -# Special type subject that denotes 'insert userid'. -RLS_USERID_SUBJECT_NAME = "userid" -RLS_USERID_SUBJECT = RLSSubject(subject_type=RLSSubjectType.userid, subject_id="", subject_name=RLS_USERID_SUBJECT_NAME) - - -@attr.s(slots=True) -class RLSEntry: - field_guid: str = attr.ib() - allowed_value: Optional[str] = attr.ib() - subject: RLSSubject = attr.ib() - # Note: this is a bit of a hack to avoid very extensive splitting of the - # RLSEntry into multiple classes. - # For `pattern_type=RLSPatternType.all`, `allowed_value` must be `None`. - # For `pattern_type=RLSPatternType.userid`, `allowed_value` must be `None`, - # and `subject` must be `RLSSubjectType.userid`. - pattern_type: RLSPatternType = attr.ib(default=RLSPatternType.value) - - def ensure_removed_failed_subject_prefix(self) -> RLSEntry: - rls_entry = copy.deepcopy(self) - rls_entry.subject.subject_name = rls_entry.subject.subject_name.removeprefix(RLS_FAILED_USER_NAME_PREFIX) - return rls_entry +class FieldRestrictions(NamedTuple): + allow_all_values: bool + allow_userid: bool + allowed_values: list[str] @attr.s class RLS: - items: List[RLSEntry] = attr.ib(factory=list) + items: list[RLSEntry] = attr.ib(factory=list) @property def has_restrictions(self) -> bool: return bool(self.items) @property - def fields_with_rls(self) -> List[str]: + def fields_with_rls(self) -> list[str]: return list(set(item.field_guid for item in self.items)) def get_entries( self, field_guid: str, subject_type: RLSSubjectType, subject_id: str, add_userid_entry: bool = True - ) -> List[RLSEntry]: + ) -> list[RLSEntry]: return [ item for item in self.items @@ -94,7 +61,7 @@ def get_field_restriction_for_subject( field_guid: str, subject_type: RLSSubjectType, subject_id: str, - ) -> Tuple[bool, bool, Optional[List[str]]]: + ) -> FieldRestrictions: """ For subject and field, return `allow_all_values, allowed_values`. """ @@ -102,37 +69,42 @@ def get_field_restriction_for_subject( # There's a `*: {current_user}` entry, no need to filter. if any(rls_entry.pattern_type == RLSPatternType.all for rls_entry in rls_entries): - return True, False, None + return FieldRestrictions(allow_all_values=True, allow_userid=False, allowed_values=[]) # Pick out userid-entry, if any - userid_entries, rls_entries = split_list( - rls_entries, lambda rls_entry: rls_entry.pattern_type == RLSPatternType.userid - ) + userid_entry: Optional[RLSEntry] = None + for rls_entry in rls_entries: + if rls_entry.pattern_type != RLSPatternType.userid: + continue + if userid_entry is not None: + raise ValueError("Expected no more than one userid entries") + userid_entry = rls_entry + if userid_entry is not None: + rls_entries.remove(userid_entry) # normal values assert all( rls_entry.pattern_type == RLSPatternType.value for rls_entry in rls_entries ), "only simple values should remain at this point" - allowed_values = [rls_entry.allowed_value for rls_entry in rls_entries] + allowed_values = cast(list[str], [rls_entry.allowed_value for rls_entry in rls_entries]) # cast for mypy + assert all(value is not None for value in allowed_values) # `userid: userid` case allow_userid = False - if userid_entries: - assert len(userid_entries) == 1 - userid_entry = userid_entries[0] + if userid_entry is not None: assert userid_entry.pattern_type == RLSPatternType.userid assert userid_entry.allowed_value is None # only `userid: userid` makes sense assert userid_entry.subject.subject_type == RLSSubjectType.userid allow_userid = True - return False, allow_userid, allowed_values # type: ignore # TODO: fix + return FieldRestrictions(allow_all_values=False, allow_userid=allow_userid, allowed_values=allowed_values) def get_subject_restrictions( self, subject_type: RLSSubjectType, subject_id: str, - ) -> Dict[str, List[str]]: + ) -> dict[str, list[str]]: result = {} for field_guid in self.fields_with_rls: allow_all_values, allow_userid, allowed_values = self.get_field_restriction_for_subject( @@ -144,14 +116,8 @@ def get_subject_restrictions( # For `userid: userid`, add the subject id to the values. if allow_userid: - allowed_values = list(allowed_values) + [subject_id] # type: ignore # TODO: fix + allowed_values = list(allowed_values) + [subject_id] result[field_guid] = allowed_values - return result # type: ignore # TODO: fix - - -class BaseSubjectResolver(metaclass=abc.ABCMeta): - @abc.abstractmethod - def get_subjects_by_names(self, names: List[str]) -> List[RLSSubject]: - raise NotImplementedError + return result diff --git a/lib/dl_api_lib/dl_api_lib/utils/rls.py b/lib/dl_rls/dl_rls/serializer.py similarity index 97% rename from lib/dl_api_lib/dl_api_lib/utils/rls.py rename to lib/dl_rls/dl_rls/serializer.py index a41190ab6..77fe3ccd8 100644 --- a/lib/dl_api_lib/dl_api_lib/utils/rls.py +++ b/lib/dl_rls/dl_rls/serializer.py @@ -17,24 +17,24 @@ import attr import more_itertools -from dl_api_lib import exc -from dl_api_lib.utils.base import ( - chunks, - quote_by_quote, - split_by_quoted_quote, -) from dl_constants.enums import RLSSubjectType -from dl_core.rls import ( +from dl_rls import exc +from dl_rls.models import ( RLS_ALL_SUBJECT, RLS_ALL_SUBJECT_NAME, RLS_FAILED_USER_NAME_PREFIX, RLS_USERID_SUBJECT, RLS_USERID_SUBJECT_NAME, - BaseSubjectResolver, RLSEntry, RLSPatternType, RLSSubject, ) +from dl_rls.subject_resolver import BaseSubjectResolver +from dl_rls.utils import ( + chunks, + quote_by_quote, + split_by_quoted_quote, +) LOGGER = logging.getLogger(__name__) @@ -129,13 +129,13 @@ def _parse_single_line(cls, line: str) -> Tuple[RLSPatternType, Optional[str], L # Some extra validation. if cls.allow_all_subject_name in subject_names: if pattern_type == RLSPatternType.all: - raise ValueError(("Wildcard `*: *` is not allowed." " It would effectively disable RLS for the field.")) + raise ValueError("Wildcard `*: *` is not allowed. It would effectively disable RLS for the field.") if len(subject_names) != 1: # Note that this does not check for # value1: * # value1: user, … # lines. - raise ValueError(("Wildcard user must be the only user in line" ", i.e. `…: *`.")) + raise ValueError("Wildcard user must be the only user in line, i.e. `…: *`.") return pattern_type, value, subject_names diff --git a/lib/dl_rls/dl_rls/subject_resolver.py b/lib/dl_rls/dl_rls/subject_resolver.py new file mode 100644 index 000000000..60c70ae91 --- /dev/null +++ b/lib/dl_rls/dl_rls/subject_resolver.py @@ -0,0 +1,9 @@ +import abc + +from dl_rls.models import RLSSubject + + +class BaseSubjectResolver(metaclass=abc.ABCMeta): + @abc.abstractmethod + def get_subjects_by_names(self, names: list[str]) -> list[RLSSubject]: + raise NotImplementedError diff --git a/lib/dl_rls/dl_rls/utils.py b/lib/dl_rls/dl_rls/utils.py new file mode 100644 index 000000000..7718b5ef0 --- /dev/null +++ b/lib/dl_rls/dl_rls/utils.py @@ -0,0 +1,60 @@ +from typing import ( + Iterable, + TypeVar, +) + + +_T = TypeVar("_T") + + +# TODO: replace with itertools.batched after switching to Python 3.12 +def chunks(lst: list[_T], size: int) -> Iterable[list[_T]]: + """Yield successive chunks from lst. No padding.""" + for idx in range(0, len(lst), size): + yield lst[idx : idx + size] + + +def split_by_quoted_quote(value: str, quote: str = "'") -> tuple[str, str]: + """ + Parse out a quoted value at the beginning, + where quotes are quoted by doubling (CSV-like). + + >>> split_by_quoted_quote("'abc'de") + ('abc', 'de') + >>> split_by_quoted_quote("'ab''c'''de") + ("ab'c'", 'de') + >>> split_by_quoted_quote("'ab''c'''") + ("ab'c'", '') + """ + ql = len(quote) + if not value.startswith(quote): + raise ValueError("Value does not start with quote") + value = value[ql:] + result = [] + while True: + try: + next_quote = value.index(quote) + except ValueError as e: + raise ValueError("Unclosed quote") from e + value_piece = value[:next_quote] + result.append(value_piece) + value = value[next_quote + ql :] + if value.startswith(quote): + result.append(quote) + value = value[ql:] + else: # some other text, or end-of-line. + break + + return "".join(result), value + + +def quote_by_quote(value: str, quote: str = "'") -> str: + """ + Inverse function for split_by_quoted_quote + + >>> quote_by_quote("a'b'") + "'a''b'''" + >>> split_by_quoted_quote(quote_by_quote("a'b'") + "and 'stuff'") + ("a'b'", "and 'stuff'") + """ + return "{}{}{}".format(quote, value.replace(quote, quote + quote), quote) diff --git a/lib/dl_rls/dl_rls_tests/__init__.py b/lib/dl_rls/dl_rls_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_rls/dl_rls_tests/unit/__init__.py b/lib/dl_rls/dl_rls_tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_rls/dl_rls_tests/unit/conftest.py b/lib/dl_rls/dl_rls_tests/unit/conftest.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_rls/pyproject.toml b/lib/dl_rls/pyproject.toml new file mode 100644 index 000000000..18babc28a --- /dev/null +++ b/lib/dl_rls/pyproject.toml @@ -0,0 +1,39 @@ + +[tool.poetry] +name = "datalens-rls" +version = "0.0.1" +description = "" +authors = ["DataLens Team "] +packages = [{include = "dl_rls"}] +license = "Apache 2.0" +readme = "README.md" + + +[tool.poetry.dependencies] +attrs = ">=22.2.0" +more-itertools = ">=9.1.0" +python = ">=3.10, <3.13" +datalens-constants = {path = "../../lib/dl_constants"} + +[tool.poetry.group.tests.dependencies] +pytest = ">=7.2.2" + +[build-system] +build-backend = "poetry.core.masonry.api" +requires = [ + "poetry-core", +] + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra" +testpaths = [] + +[datalens_ci] +skip_test = true + +[tool.mypy] +warn_unused_configs = true +disallow_untyped_defs = true +check_untyped_defs = true +strict_optional = true diff --git a/lib/dl_utils/dl_utils/utils.py b/lib/dl_utils/dl_utils/utils.py index 96831eaa2..a97964a0a 100644 --- a/lib/dl_utils/dl_utils/utils.py +++ b/lib/dl_utils/dl_utils/utils.py @@ -1,19 +1,13 @@ from __future__ import annotations -from contextlib import contextmanager from enum import Enum import functools from itertools import islice import operator -from time import time from typing import ( Any, - Callable, - Generator, Iterable, - List, Optional, - Tuple, Type, TypeVar, ) @@ -30,26 +24,6 @@ def get_type_full_name(t: Type) -> str: return f"{module}.{qual_name}" -# split_list value TypeVar -_SL_V_TV = TypeVar("_SL_V_TV") - - -def split_list( - iterable: Iterable[_SL_V_TV], condition: Callable[[_SL_V_TV], bool] -) -> Tuple[List[_SL_V_TV], List[_SL_V_TV]]: - """ - Split list items into `(matching, non_matching)` by `condition(item)` callable. - """ - matching: List[_SL_V_TV] = [] - non_matching: List[_SL_V_TV] = [] - for item in iterable: - if condition(item): - matching.append(item) - else: - non_matching.append(item) - return matching, non_matching - - class DotDict(dict): """A simple dict subclass with items also available over attributes""" @@ -127,30 +101,6 @@ def enum_not_none(val: Optional[_ENUM_TV]) -> _ENUM_TV: return val -def time_it(fn: Callable) -> Callable: - @functools.wraps(fn) - def wrap(*args: Any, **kwargs: Any) -> Any: - t0 = time() - print(f"Invoked {fn}({args}, {kwargs})"[:160]) - result = fn(*args, **kwargs) - delta = time() - t0 - if delta >= 0.01: - print(f"<< Time elapsed: {delta} for {fn}({args}, {kwargs})"[:160]) - return result - - return wrap - - -@contextmanager -def time_it_cm(label: str) -> Generator[None, None, None]: - # print(f'Time it for {label}') - t0 = time() - yield - delta = time() - t0 - if delta >= 0.01: - print(f"Time elapsed for {label}: {delta}") - - def make_url( protocol: str, host: str, diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index 7e9cb9af3..2168fcea2 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -1032,6 +1032,7 @@ datalens-model-tools = {path = "../dl_model_tools"} datalens-pivot = {path = "../dl_pivot"} datalens-pivot-pandas = {path = "../dl_pivot_pandas"} datalens-query-processing = {path = "../dl_query_processing"} +datalens-rls = {path = "../dl_rls"} datalens-task-processor = {path = "../dl_task_processor"} datalens-utils = {path = "../dl_utils"} flask = ">=2.2.5" @@ -1039,7 +1040,6 @@ flask-marshmallow = ">=1.1.0" flask-restx = ">=1.1.0" marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" -more-itertools = ">=9.1.0" statcommons = {path = "../statcommons"} werkzeug = ">=2.2.3" @@ -1072,6 +1072,7 @@ datalens-formula-testing = {path = "../dl_formula_testing"} datalens-i18n = {path = "../dl_i18n"} datalens-pivot-pandas = {path = "../dl_pivot_pandas"} datalens-query-processing = {path = "../dl_query_processing"} +datalens-rls = {path = "../dl_rls"} datalens-testing = {path = "../dl_testing"} datalens-utils = {path = "../dl_utils"} flask = ">=2.2.5" @@ -1719,6 +1720,7 @@ datalens-constants = {path = "../dl_constants"} datalens-dashsql = {path = "../../lib/dl_dashsql"} datalens-i18n = {path = "../dl_i18n"} datalens-model-tools = {path = "../dl_model_tools"} +datalens-rls = {path = "../dl_rls"} datalens-task-processor = {path = "../dl_task_processor"} datalens-utils = {path = "../dl_utils"} dnspython = ">=2.2.1" @@ -2218,6 +2220,24 @@ flask = ["flask (>=2.2.5)"] type = "directory" url = "../lib/dl_rate_limiter" +[[package]] +name = "datalens-rls" +version = "0.0.1" +description = "" +optional = false +python-versions = ">=3.10, <3.13" +files = [] +develop = false + +[package.dependencies] +attrs = ">=22.2.0" +datalens-constants = {path = "../../lib/dl_constants"} +more-itertools = ">=9.1.0" + +[package.source] +type = "directory" +url = "../lib/dl_rls" + [[package]] name = "datalens-sqlalchemy-bitrix" version = "0.0.1" @@ -6608,4 +6628,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.13" -content-hash = "e885bd42bc2805174b0d15d6016baa3e0b9eb81da42058a7d80a1bc8d70f7710" +content-hash = "35f54cf163d2eea1e130e1311802fdca170e5465e3aaca66bedb81a4c3c90b8c" diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index 0d7681e5c..a06b390b4 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -153,6 +153,7 @@ datalens-pivot = {path = "../lib/dl_pivot"} datalens-pivot-pandas = {path = "../lib/dl_pivot_pandas"} datalens-cache-engine = {path = "../lib/dl_cache_engine"} datalens-rate-limiter = {path = "../lib/dl_rate_limiter"} +datalens-rls = {path = "../lib/dl_rls"} [tool.poetry.group.dev.dependencies] black = "==23.12.1"