diff --git a/src/fides/api/api/deps.py b/src/fides/api/api/deps.py index 8de85d21d7..397fb40da8 100644 --- a/src/fides/api/api/deps.py +++ b/src/fides/api/api/deps.py @@ -10,6 +10,7 @@ from fides.config import CONFIG, FidesConfig from fides.config import get_config as get_app_config from fides.config.config_proxy import ConfigProxy +from fides.service.dataset.dataset_config_service import DatasetConfigService from fides.service.dataset.dataset_service import DatasetService from fides.service.messaging.messaging_service import MessagingService from fides.service.privacy_request.privacy_request_service import PrivacyRequestService @@ -89,3 +90,7 @@ def get_privacy_request_service( def get_dataset_service(db: Session = Depends(get_db)) -> DatasetService: return DatasetService(db) + + +def get_dataset_config_service(db: Session = Depends(get_db)) -> DatasetConfigService: + return DatasetConfigService(db) diff --git a/src/fides/api/api/v1/api.py b/src/fides/api/api/v1/api.py index 527a2a64e4..79e92d4b81 100644 --- a/src/fides/api/api/v1/api.py +++ b/src/fides/api/api/v1/api.py @@ -3,7 +3,7 @@ connection_endpoints, connection_type_endpoints, consent_request_endpoints, - dataset_endpoints, + dataset_config_endpoints, drp_endpoints, encryption_endpoints, identity_verification_endpoints, @@ -28,7 +28,7 @@ api_router.include_router(connection_type_endpoints.router) api_router.include_router(connection_endpoints.router) api_router.include_router(consent_request_endpoints.router) -api_router.include_router(dataset_endpoints.router) +api_router.include_router(dataset_config_endpoints.router) api_router.include_router(drp_endpoints.router) api_router.include_router(encryption_endpoints.router) api_router.include_router(masking_endpoints.router) diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_config_endpoints.py similarity index 92% rename from src/fides/api/api/v1/endpoints/dataset_endpoints.py rename to src/fides/api/api/v1/endpoints/dataset_config_endpoints.py index 2b29a090d4..81ced9942c 100644 --- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/api/v1/endpoints/dataset_config_endpoints.py @@ -4,7 +4,6 @@ from fastapi import Depends, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.params import Security -from fastapi.responses import JSONResponse from fastapi_pagination import Page, Params from fastapi_pagination.bases import AbstractPage from fastapi_pagination.ext.sqlalchemy import paginate @@ -55,17 +54,16 @@ DATASET_REACHABILITY, DATASET_VALIDATE, DATASETS, - DATASETS_CLEAN, TEST_DATASET, V1_URL_PREFIX, YAML_DATASETS, ) from fides.config import CONFIG -from fides.service.dataset.dataset_service import ( - DatasetNotFoundException, - DatasetService, +from fides.service.dataset.dataset_config_service import ( + DatasetConfigService, get_identities_and_references, ) +from fides.service.dataset.dataset_service import DatasetNotFoundException from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip Dataset as CtlDataset, @@ -98,7 +96,9 @@ def _get_connection_config( ) def validate_dataset( dataset: FideslangDataset, - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> ValidateDatasetResponse: """ @@ -118,7 +118,9 @@ def validate_dataset( """ try: - return dataset_service.validate_dataset(connection_config, dataset) + return dataset_config_service.validate_dataset_config( + connection_config, dataset + ) except PydanticValidationError as e: raise HTTPException( status_code=HTTP_422_UNPROCESSABLE_ENTITY, @@ -135,7 +137,9 @@ def validate_dataset( def put_dataset_configs( dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)], # type: ignore db: Session = Depends(deps.get_db), - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> BulkPutDataset: """ @@ -171,7 +175,9 @@ def put_dataset_configs( db.commit() # reuse the existing patch logic once we've removed the unused dataset configs - return patch_dataset_configs(dataset_pairs, dataset_service, connection_config) + return patch_dataset_configs( + dataset_pairs, dataset_config_service, connection_config + ) @router.patch( @@ -182,7 +188,9 @@ def put_dataset_configs( ) def patch_dataset_configs( dataset_pairs: Annotated[List[DatasetConfigCtlDataset], Field(max_length=50)], # type: ignore - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> BulkPutDataset: """ @@ -194,7 +202,7 @@ def patch_dataset_configs( to the DatasetConfig. """ try: - return dataset_service.bulk_create_or_update_datasets( + return dataset_config_service.bulk_create_or_update_dataset_configs( connection_config, dataset_pairs, ) @@ -218,7 +226,9 @@ def patch_dataset_configs( ) def patch_datasets( datasets: Annotated[List[FideslangDataset], Field(max_length=50)], # type: ignore - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> BulkPutDataset: """ @@ -232,7 +242,7 @@ def patch_datasets( """ try: - return dataset_service.bulk_create_or_update_datasets( + return dataset_config_service.bulk_create_or_update_dataset_configs( connection_config, datasets ) except PydanticValidationError as e: @@ -250,7 +260,9 @@ def patch_datasets( ) async def patch_yaml_datasets( request: Request, - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), connection_config: ConnectionConfig = Depends(_get_connection_config), ) -> BulkPutDataset: """ @@ -274,7 +286,7 @@ async def patch_yaml_datasets( for dataset_dict in yaml_request_body["dataset"] ] - return dataset_service.bulk_create_or_update_datasets( + return dataset_config_service.bulk_create_or_update_dataset_configs( connection_config, datasets ) @@ -497,29 +509,6 @@ def get_ctl_datasets( return datasets -@router.get( - DATASETS_CLEAN, - dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], - response_model=List[FideslangDataset], - deprecated=True, -) -def clean_datasets( - dataset_service: DatasetService = Depends(deps.get_dataset_service), -) -> JSONResponse: - """ - Clean up names of datasets and upsert them. - """ - - succeeded, failed = dataset_service.clean_datasets() - return JSONResponse( - status_code=HTTP_200_OK, - content={ - "succeeded": succeeded, - "failed": failed, - }, - ) - - @router.get( DATASET_INPUTS, dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], @@ -561,7 +550,9 @@ def dataset_reachability( *, db: Session = Depends(deps.get_db), connection_config: ConnectionConfig = Depends(_get_connection_config), - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), dataset_key: FidesKey, policy_key: Optional[FidesKey] = None, ) -> Dict[str, Any]: @@ -592,7 +583,7 @@ def dataset_reachability( detail=f'Policy with key "{policy_key}" not found', ) - reachable, details = dataset_service.get_dataset_reachability( + reachable, details = dataset_config_service.get_dataset_reachability( dataset_config, access_policy ) return {"reachable": reachable, "details": details} @@ -607,7 +598,9 @@ def dataset_reachability( def test_connection_datasets( *, db: Session = Depends(deps.get_db), - dataset_service: DatasetService = Depends(deps.get_dataset_service), + dataset_config_service: DatasetConfigService = Depends( + deps.get_dataset_config_service + ), connection_config: ConnectionConfig = Depends(_get_connection_config), dataset_key: FidesKey, test_request: DatasetTestRequest, @@ -639,7 +632,7 @@ def test_connection_datasets( detail=f'Policy with key "{test_request.policy_key}" not found', ) - privacy_request = dataset_service.run_test_access_request( + privacy_request = dataset_config_service.run_test_access_request( access_policy, dataset_config, input_data=test_request.identities.data, diff --git a/src/fides/api/api/v1/endpoints/generic_overrides.py b/src/fides/api/api/v1/endpoints/generic_overrides.py index 20123ede9b..6c01610694 100644 --- a/src/fides/api/api/v1/endpoints/generic_overrides.py +++ b/src/fides/api/api/v1/endpoints/generic_overrides.py @@ -1,17 +1,23 @@ from typing import Dict, List, Optional, Type, Union from fastapi import APIRouter, Depends, HTTPException, Query, Security +from fastapi.responses import JSONResponse from fastapi_pagination import Page, Params from fastapi_pagination.ext.async_sqlalchemy import paginate as async_paginate -from fideslang.models import Dataset -from sqlalchemy import not_ +from fideslang.models import Dataset as FideslangDataset +from pydantic import ValidationError as PydanticValidationError +from sqlalchemy import not_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session -from sqlalchemy.sql.expression import select from starlette import status -from starlette.status import HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY +from starlette.status import ( + HTTP_200_OK, + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, +) -from fides.api.api.deps import get_db +from fides.api.api import deps +from fides.api.api.deps import get_dataset_service, get_db from fides.api.common_exceptions import KeyOrNameAlreadyExists from fides.api.db.base_class import get_key_from_data from fides.api.db.crud import list_resource_query @@ -31,17 +37,27 @@ from fides.api.util.errors import FidesError, ForbiddenIsDefaultTaxonomyError from fides.api.util.filter_utils import apply_filters_to_query from fides.common.api.scope_registry import ( + CTL_DATASET_CREATE, + CTL_DATASET_UPDATE, DATA_CATEGORY_CREATE, DATA_CATEGORY_UPDATE, DATA_SUBJECT_CREATE, DATA_USE_CREATE, DATA_USE_UPDATE, + DATASET_DELETE, DATASET_READ, ) -from fides.common.api.v1.urn_registry import V1_URL_PREFIX +from fides.common.api.v1.urn_registry import DATASETS_CLEAN, V1_URL_PREFIX +from fides.service.dataset.dataset_service import ( + DatasetNotFoundException, + DatasetService, +) from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip Dataset as CtlDataset, +) + +from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip DataCategory as DataCategoryDbModel, DataSubject as DataSubjectDbModel, DataUse as DataUseDbModel, @@ -57,10 +73,60 @@ data_subject_router = APIRouter(tags=["DataSubject"], prefix=V1_URL_PREFIX) +@dataset_router.post( + "/dataset", + dependencies=[Security(verify_oauth_client, scopes=[CTL_DATASET_CREATE])], + response_model=FideslangDataset, + status_code=status.HTTP_201_CREATED, + name="Create dataset", +) +async def create_dataset( + dataset: FideslangDataset, + dataset_service: DatasetService = Depends(get_dataset_service), +) -> Dict: + """Create a new dataset""" + try: + created = dataset_service.create_dataset(dataset) + return created.model_dump() + except PydanticValidationError as e: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail={"message": str(e)}, + ) + + +@dataset_router.put( + "/dataset", + dependencies=[Security(verify_oauth_client, scopes=[CTL_DATASET_UPDATE])], + response_model=FideslangDataset, + status_code=status.HTTP_200_OK, + name="Update dataset", +) +async def update_dataset( + dataset: FideslangDataset, + db: Session = Depends(get_db), +) -> Dict: + """Update an existing dataset""" + service = DatasetService(db) + try: + updated = service.update_dataset(dataset) + return updated.model_dump() + except PydanticValidationError as e: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail={"message": str(e)}, + ) + except DatasetNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail={"message": str(e)}, + ) + + @dataset_router.get( "/dataset", dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], - response_model=Union[Page[Dataset], List[Dataset]], + response_model=Union[Page[FideslangDataset], List[FideslangDataset]], name="List datasets (optionally paginated)", ) async def list_dataset_paginated( @@ -71,7 +137,7 @@ async def list_dataset_paginated( data_categories: Optional[List[str]] = Query(None), exclude_saas_datasets: Optional[bool] = Query(False), only_unlinked_datasets: Optional[bool] = Query(False), -) -> Union[Page[Dataset], List[Dataset]]: +) -> Union[Page[FideslangDataset], List[FideslangDataset]]: """ Get a list of all of the Datasets. If any pagination parameters (size or page) are provided, then the response will be paginated. @@ -114,6 +180,72 @@ async def list_dataset_paginated( return await async_paginate(db, filtered_query, pagination_params) +@dataset_router.get( + "/dataset/{fides_key}", + dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], + response_model=FideslangDataset, + name="Get dataset", +) +async def get_dataset( + fides_key: str, + db: Session = Depends(get_db), +) -> Dict: + """Get a single dataset by fides key""" + service = DatasetService(db) + try: + dataset = service.get_dataset(fides_key) + return dataset.model_dump() + except DatasetNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail={"message": str(e)}, + ) + + +@dataset_router.delete( + "/dataset/{fides_key}", + dependencies=[Security(verify_oauth_client, scopes=[DATASET_DELETE])], + status_code=status.HTTP_204_NO_CONTENT, + name="Delete dataset", +) +async def delete_dataset( + fides_key: str, + db: Session = Depends(get_db), +) -> None: + """Delete a dataset by fides key""" + service = DatasetService(db) + try: + service.delete_dataset(fides_key) + except DatasetNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail={"message": str(e)}, + ) + + +@dataset_router.get( + DATASETS_CLEAN, + dependencies=[Security(verify_oauth_client, scopes=[DATASET_READ])], + response_model=List[FideslangDataset], + deprecated=True, +) +def clean_datasets( + dataset_service: DatasetService = Depends(deps.get_dataset_service), +) -> JSONResponse: + """ + Clean up names of datasets and upsert them. + """ + + succeeded, failed = dataset_service.clean_datasets() + return JSONResponse( + status_code=HTTP_200_OK, + content={ + "succeeded": succeeded, + "failed": failed, + }, + ) + + def activate_taxonomy_parents( resource: Union[DataCategoryDbModel, DataUseDbModel, DataSubjectDbModel], db: Session, diff --git a/src/fides/api/api/v1/endpoints/manual_webhook_endpoints.py b/src/fides/api/api/v1/endpoints/manual_webhook_endpoints.py index 6a72aa17e2..3ce8c5f41d 100644 --- a/src/fides/api/api/v1/endpoints/manual_webhook_endpoints.py +++ b/src/fides/api/api/v1/endpoints/manual_webhook_endpoints.py @@ -15,7 +15,7 @@ ) from fides.api.api import deps -from fides.api.api.v1.endpoints.dataset_endpoints import _get_connection_config +from fides.api.api.v1.endpoints.dataset_config_endpoints import _get_connection_config from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType from fides.api.models.manual_webhook import AccessManualWebhook from fides.api.oauth.utils import verify_oauth_client diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index 6de122f219..d425dc663b 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -42,7 +42,7 @@ from fides.api.api import deps from fides.api.api.deps import get_privacy_request_service -from fides.api.api.v1.endpoints.dataset_endpoints import _get_connection_config +from fides.api.api.v1.endpoints.dataset_config_endpoints import _get_connection_config from fides.api.api.v1.endpoints.manual_webhook_endpoints import ( get_access_manual_webhook_or_404, ) @@ -164,7 +164,9 @@ ) from fides.config import CONFIG from fides.config.config_proxy import ConfigProxy -from fides.service.dataset.dataset_service import replace_references_with_identities +from fides.service.dataset.dataset_config_service import ( + replace_references_with_identities, +) from fides.service.messaging.messaging_service import MessagingService from fides.service.privacy_request.privacy_request_service import ( PrivacyRequestService, diff --git a/src/fides/api/api/v1/endpoints/router_factory.py b/src/fides/api/api/v1/endpoints/router_factory.py index 48d8f75c25..bed6f15743 100644 --- a/src/fides/api/api/v1/endpoints/router_factory.py +++ b/src/fides/api/api/v1/endpoints/router_factory.py @@ -10,7 +10,6 @@ from fastapi.encoders import jsonable_encoder from fideslang import FidesModelType from fideslang.models import Dataset -from fideslang.validation import FidesKey from pydantic import ValidationError as PydanticValidationError from sqlalchemy.ext.asyncio import AsyncSession from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY @@ -25,12 +24,7 @@ upsert_resources, ) from fides.api.db.ctl_session import get_async_db -from fides.api.models.datasetconfig import validate_masking_strategy_override -from fides.api.models.sql_models import ( - DataCategory, - ModelWithDefaultField, - sql_model_map, -) +from fides.api.models.sql_models import ModelWithDefaultField, sql_model_map from fides.api.oauth.utils import verify_oauth_client_prod from fides.api.util import errors from fides.api.util.api_router import APIRouter @@ -42,50 +36,16 @@ forbid_if_editing_is_default, ) from fides.common.api.scope_registry import CREATE, DELETE, READ, UPDATE -from fides.service.dataset.validation_steps.data_category import ( - validate_data_categories_against_db, -) - - -async def get_data_categories_from_db(async_session: AsyncSession) -> List[FidesKey]: - """Similar method to one on the ops side except this uses an async session to retrieve data categories""" - resources = await list_resource(DataCategory, async_session) - data_categories = [res.fides_key for res in resources] - return data_categories - - -async def validate_data_categories( - dataset: Dataset, async_session: AsyncSession -) -> None: - """ - Validate DataCategories on Datasets based on existing DataCategories in the database. - """ - try: - defined_data_categories: List[FidesKey] = await get_data_categories_from_db( - async_session - ) - validate_data_categories_against_db(dataset, defined_data_categories) - except PydanticValidationError as e: - raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - detail=jsonable_encoder(e.errors(include_url=False, include_input=False)), - ) - - -def validate_masking_strategy(dataset: Dataset) -> None: - try: - validate_masking_strategy_override(dataset) - except ValidationError as e: - raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - detail=jsonable_encoder(e.message), - ) +from fides.service.dataset.dataset_service import DatasetService def generic_router_factory(fides_model: FidesModelType, model_type: str) -> APIRouter: """ Compose all of the individual route factories into a single coherent Router. + Skip dataset-specific routes as they are now handled by DatasetService. """ + if model_type == "dataset": + return APIRouter() # Return empty router for datasets object_router = APIRouter() @@ -158,8 +118,19 @@ async def create( """ sql_model = sql_model_map[model_type] if isinstance(resource, Dataset): - await validate_data_categories(resource, db) - validate_masking_strategy(resource) + try: + dataset_service = DatasetService(db) + dataset_service.validate_dataset(resource) + except (ValidationError, PydanticValidationError) as e: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder( + e.errors(include_url=False, include_input=False) + if isinstance(e, PydanticValidationError) + else e.message + ), + ) + if isinstance(sql_model, ModelWithDefaultField) and resource.is_default: raise errors.ForbiddenIsDefaultTaxonomyError( model_type, resource.fides_key, action="create" @@ -264,8 +235,18 @@ async def update( """ sql_model = sql_model_map[model_type] if isinstance(resource, Dataset): - await validate_data_categories(resource, db) - validate_masking_strategy(resource) + try: + dataset_service = DatasetService(db) + dataset_service.validate_dataset(resource) + except (ValidationError, PydanticValidationError) as e: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder( + e.errors(include_url=False, include_input=False) + if isinstance(e, PydanticValidationError) + else e.message + ), + ) await forbid_if_editing_is_default(sql_model, resource.fides_key, resource, db) return await update_resource(sql_model, resource.model_dump(mode="json"), db) @@ -344,10 +325,21 @@ async def upsert( sql_model = sql_model_map[model_type] resource_dicts = [resource.model_dump(mode="json") for resource in resources] - for resource in resources: - if isinstance(resource, Dataset): - await validate_data_categories(resource, db) - validate_masking_strategy(resource) + if any(isinstance(resource, Dataset) for resource in resources): + try: + dataset_service = DatasetService(db) + for resource in resources: + if isinstance(resource, Dataset): + dataset_service.validate_dataset(resource) + except (ValidationError, PydanticValidationError) as e: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=jsonable_encoder( + e.errors(include_url=False, include_input=False) + if isinstance(e, PydanticValidationError) + else e.message + ), + ) await forbid_if_editing_any_is_default(sql_model, resource_dicts, db) result = await upsert_resources(sql_model, resource_dicts, db) response.status_code = ( diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index 78e3e7715c..9092c4c1b9 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session -from fides.api.api.v1.endpoints.dataset_endpoints import patch_dataset_configs +from fides.api.api.v1.endpoints.dataset_config_endpoints import patch_dataset_configs from fides.api.api.v1.endpoints.saas_config_endpoints import ( instantiate_connection_from_template, ) @@ -42,7 +42,7 @@ from fides.api.util.errors import AlreadyExistsError, QueryError from fides.api.util.text import to_snake_case from fides.config import CONFIG -from fides.service.dataset.dataset_service import DatasetService +from fides.service.dataset.dataset_config_service import DatasetConfigService from .crud import create_resource, get_resource, list_resource, upsert_resources from .samples import ( @@ -503,10 +503,10 @@ async def load_samples(async_session: AsyncSession) -> None: dataset_pair = DatasetConfigCtlDataset( fides_key=dataset_key, ctl_dataset_fides_key=dataset_key ) - dataset_service = DatasetService(db=db_session) + dataset_config_service = DatasetConfigService(db=db_session) patch_dataset_configs( dataset_pairs=[dataset_pair], - dataset_service=dataset_service, + dataset_config_service=dataset_config_service, connection_config=connection_config, ) dataset_config = DatasetConfig.get_by( diff --git a/src/fides/service/dataset/dataset_config_service.py b/src/fides/service/dataset/dataset_config_service.py new file mode 100644 index 0000000000..031e8ddd6c --- /dev/null +++ b/src/fides/service/dataset/dataset_config_service.py @@ -0,0 +1,306 @@ +from copy import deepcopy +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +from fideslang.models import Dataset as FideslangDataset +from fideslang.validation import FidesKey +from loguru import logger +from pydantic import ValidationError as PydanticValidationError +from sqlalchemy.orm import Session + +from fides.api.common_exceptions import ( + SaaSConfigNotFoundException, + UnreachableNodesError, + ValidationError, +) +from fides.api.graph.config import GraphDataset +from fides.api.graph.graph import DatasetGraph +from fides.api.graph.traversal import Traversal +from fides.api.models.connectionconfig import ConnectionConfig +from fides.api.models.datasetconfig import DatasetConfig +from fides.api.models.policy import Policy +from fides.api.models.privacy_request import ( + PrivacyRequest, + PrivacyRequestSource, + PrivacyRequestStatus, +) +from fides.api.schemas.api import BulkUpdateFailed +from fides.api.schemas.dataset import ( + BulkPutDataset, + DatasetConfigCtlDataset, + ValidateDatasetResponse, +) +from fides.api.schemas.redis_cache import Identity, LabeledIdentity +from fides.api.util.data_category import get_data_categories_from_db +from fides.service.dataset.dataset_service import ( + DatasetNotFoundException, + _get_ctl_dataset, +) +from fides.service.dataset.dataset_validator import DatasetValidator +from fides.service.dataset.validation_steps.data_category import ( + validate_data_categories_against_db, +) +from fides.service.dataset.validation_steps.traversal import TraversalValidationStep + + +class DatasetConfigService: + def __init__(self, db: Session): + self.db = db + + def create_or_update_dataset_config( + self, + connection_config: ConnectionConfig, + dataset: Union[DatasetConfigCtlDataset, FideslangDataset], + ) -> Tuple[Optional[FideslangDataset], Optional[BulkUpdateFailed]]: + """Create or update a single dataset""" + try: + if isinstance(dataset, DatasetConfigCtlDataset): + ctl_dataset = _get_ctl_dataset(self.db, dataset.ctl_dataset_fides_key) + dataset_to_validate = FideslangDataset.model_validate(ctl_dataset) + data_dict = { + "connection_config_id": connection_config.id, + "fides_key": dataset.fides_key, + "ctl_dataset_id": ctl_dataset.id, + } + else: + dataset_to_validate = dataset + data_dict = { + "connection_config_id": connection_config.id, + "fides_key": dataset.fides_key, + "dataset": dataset.model_dump(mode="json"), + } + + # Validate dataset + DatasetValidator( + self.db, + dataset_to_validate, + connection_config, + skip_steps=[TraversalValidationStep], + ).validate() + + # Create or update using unified method + dataset_config = DatasetConfig.upsert_with_ctl_dataset( + self.db, data=data_dict + ) + + return dataset_config.ctl_dataset, None + + except (SaaSConfigNotFoundException, ValidationError) as exception: + error = BulkUpdateFailed( + message=str(exception), + data=dataset.model_dump(), + ) + logger.warning(f"Dataset validation failed: {str(exception)}") + return None, error + + except (PydanticValidationError, DatasetNotFoundException): + raise + + except Exception as e: + message = f"Create/update failed for dataset '{dataset.fides_key}'" + logger.warning(f"{message}: {str(e)}") + error = BulkUpdateFailed( + message="Dataset create/update failed.", + data=dataset.model_dump(), + ) + return None, error + + def bulk_create_or_update_dataset_configs( + self, + connection_config: ConnectionConfig, + datasets: Union[List[DatasetConfigCtlDataset], List[FideslangDataset]], + ) -> BulkPutDataset: + """Create or update multiple datasets""" + created_or_updated: List[FideslangDataset] = [] + failed: List[BulkUpdateFailed] = [] + + logger.info("Starting bulk upsert for {} datasets", len(datasets)) + + for item in datasets: + dataset_result, error = self.create_or_update_dataset_config( + connection_config=connection_config, + dataset=item, + ) + + if dataset_result: + created_or_updated.append(dataset_result) + if error: + failed.append(error) + + return BulkPutDataset( + succeeded=created_or_updated, + failed=failed, + ) + + def validate_dataset_config( + self, connection_config: ConnectionConfig, dataset: FideslangDataset + ) -> ValidateDatasetResponse: + + return DatasetValidator(self.db, dataset, connection_config).validate() + + def get_dataset_reachability( + self, dataset_config: DatasetConfig, policy: Optional[Policy] = None + ) -> Tuple[bool, Optional[str]]: + """ + Determines if the given dataset is reachable along with an error message + """ + + # Get all the dataset configs that are not associated with a disabled connection + datasets = DatasetConfig.all(db=self.db) + dataset_graphs = [ + dataset_config.get_graph() + for dataset_config in datasets + if not dataset_config.connection_config.disabled + ] + + # We still want to check reachability even if our dataset config's connection is disabled. + # We also consider the siblings, because if the connection is enabled, then all the + # datasets will be enabled with it. + sibling_dataset_graphs = [] + if dataset_config.connection_config.disabled: + sibling_datasets = dataset_config.connection_config.datasets + sibling_dataset_graphs = [ + dataset_config.get_graph() for dataset_config in sibling_datasets + ] + + try: + dataset_graph = DatasetGraph(*dataset_graphs, *sibling_dataset_graphs) + except ValidationError as exc: + return ( + False, + f'The following dataset references do not exist "{", ".join(exc.errors)}"', + ) + + # dummy data is enough to determine traversability + identity_seed: Dict[str, str] = { + k: "something" for k in dataset_graph.identity_keys.values() + } + + try: + Traversal(dataset_graph, identity_seed, policy) + except UnreachableNodesError as exc: + return ( + False, + f'The following collections are not reachable "{", ".join(exc.errors)}"', + ) + + return True, None + + def run_test_access_request( + self, + policy: Policy, + dataset_config: DatasetConfig, + input_data: Dict[str, Any], + ) -> PrivacyRequest: + """ + Creates a privacy request with a source of "Dataset test" that runs an access request for a single dataset. + The input data is used to mock any external dataset values referenced by our dataset so that it can run in + complete isolation. + """ + + # Create a privacy request with a source of "Dataset test" + # so it's not shown to the user. + privacy_request = PrivacyRequest.create( + db=self.db, + data={ + "policy_id": policy.id, + "source": PrivacyRequestSource.dataset_test, + "status": PrivacyRequestStatus.in_processing, + }, + ) + + # Remove periods and colons to avoid them being parsed as path delimiters downstream. + escaped_input_data = { + key.replace(".", "_").replace(":", "_"): value + for key, value in input_data.items() + } + + # Manually cache the input data as identity data. + # We're doing a bit of trickery here to avoid asking for labels for custom identities. + predefined_fields = Identity.model_fields.keys() + input_identity = { + key: ( + value + if key in predefined_fields + else LabeledIdentity(label=key, value=value) + ) + for key, value in escaped_input_data.items() + } + privacy_request.cache_identity(input_identity) + + graph_dataset = dataset_config.get_graph() + modified_graph_dataset = replace_references_with_identities( + dataset_config.fides_key, graph_dataset + ) + + dataset_graph = DatasetGraph(modified_graph_dataset) + connection_config = dataset_config.connection_config + + from fides.api.task.create_request_tasks import run_access_request + + # Finally invoke the existing DSR 3.0 access request task + run_access_request( + privacy_request, + policy, + dataset_graph, + [connection_config], + escaped_input_data, + self.db, + privacy_request_proceed=False, + ) + return privacy_request + + +def get_identities_and_references( + dataset_config: DatasetConfig, +) -> Set[str]: + """ + Returns all identity and dataset references in the dataset. + If a field has multiple references only the first reference will be considered. + """ + + result: Set[str] = set() + dataset: GraphDataset = dataset_config.get_graph() + for collection in dataset.collections: + # Process the identities in the collection + result.update(collection.identities().values()) + for _, field_refs in collection.references().items(): + # Take first reference only, we only care that this collection is reachable, + # how we get there doesn't matter for our current use case + ref, edge_direction = field_refs[0] + if edge_direction == "from" and ref.dataset != dataset_config.fides_key: + result.add(ref.value) + return result + + +def replace_references_with_identities( + dataset_key: str, graph_dataset: GraphDataset +) -> GraphDataset: + """ + Replace external field references with identity values for testing. + + Creates a copy of the graph dataset and replaces dataset references with + equivalent identity references that can be seeded directly. This allows + testing a single dataset in isolation without needing to load data from + referenced external datasets. + """ + + modified_graph_dataset = deepcopy(graph_dataset) + + for collection in modified_graph_dataset.collections: + for field in collection.fields: + for ref, edge_direction in field.references[:]: + if edge_direction == "from" and ref.dataset != dataset_key: + field.identity = f"{ref.dataset}_{ref.collection}_{'_'.join(ref.field_path.levels)}" + field.references.remove((ref, "from")) + + return modified_graph_dataset + + +def validate_data_categories(dataset: FideslangDataset, db: Session) -> None: + """Validate data categories on a given Dataset + + As a separate method because we want to be able to match against data_categories in the + database instead of a static list. + """ + defined_data_categories: List[FidesKey] = get_data_categories_from_db(db) + validate_data_categories_against_db(dataset, defined_data_categories) diff --git a/src/fides/service/dataset/dataset_service.py b/src/fides/service/dataset/dataset_service.py index 1ed875d0de..748ba91ba8 100644 --- a/src/fides/service/dataset/dataset_service.py +++ b/src/fides/service/dataset/dataset_service.py @@ -1,43 +1,13 @@ -from copy import deepcopy from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import List, Tuple from fideslang.models import Dataset as FideslangDataset -from fideslang.validation import FidesKey from loguru import logger -from pydantic import ValidationError as PydanticValidationError from sqlalchemy import select from sqlalchemy.orm import Session -from fides.api.common_exceptions import ( - SaaSConfigNotFoundException, - UnreachableNodesError, - ValidationError, -) -from fides.api.graph.config import GraphDataset -from fides.api.graph.graph import DatasetGraph -from fides.api.graph.traversal import Traversal -from fides.api.models.connectionconfig import ConnectionConfig -from fides.api.models.datasetconfig import DatasetConfig -from fides.api.models.policy import Policy -from fides.api.models.privacy_request import ( - PrivacyRequest, - PrivacyRequestSource, - PrivacyRequestStatus, -) -from fides.api.schemas.api import BulkUpdateFailed -from fides.api.schemas.dataset import ( - BulkPutDataset, - DatasetConfigCtlDataset, - ValidateDatasetResponse, -) -from fides.api.schemas.redis_cache import Identity, LabeledIdentity -from fides.api.util.data_category import get_data_categories_from_db +from fides.api.schemas.dataset import ValidateDatasetResponse from fides.service.dataset.dataset_validator import DatasetValidator -from fides.service.dataset.validation_steps.data_category import ( - validate_data_categories_against_db, -) -from fides.service.dataset.validation_steps.traversal import TraversalValidationStep from fides.api.models.sql_models import ( # type: ignore[attr-defined] # isort: skip Dataset as CtlDataset, @@ -60,268 +30,60 @@ class DatasetService: def __init__(self, db: Session): self.db = db - def create_or_update_dataset( - self, - connection_config: ConnectionConfig, - dataset: Union[DatasetConfigCtlDataset, FideslangDataset], - ) -> Tuple[Optional[FideslangDataset], Optional[BulkUpdateFailed]]: - """Create or update a single dataset""" - try: - if isinstance(dataset, DatasetConfigCtlDataset): - ctl_dataset = _get_ctl_dataset(self.db, dataset.ctl_dataset_fides_key) - dataset_to_validate = FideslangDataset.model_validate(ctl_dataset) - data_dict = { - "connection_config_id": connection_config.id, - "fides_key": dataset.fides_key, - "ctl_dataset_id": ctl_dataset.id, - } - else: - dataset_to_validate = dataset - data_dict = { - "connection_config_id": connection_config.id, - "fides_key": dataset.fides_key, - "dataset": dataset.model_dump(mode="json"), - } - - # Validate dataset - DatasetValidator( - self.db, - dataset_to_validate, - connection_config, - skip_steps=[TraversalValidationStep], - ).validate() - - # Create or update using unified method - dataset_config = DatasetConfig.upsert_with_ctl_dataset( - self.db, data=data_dict - ) - - return dataset_config.ctl_dataset, None - - except (SaaSConfigNotFoundException, ValidationError) as exception: - error = BulkUpdateFailed( - message=str(exception), - data=dataset.model_dump(), - ) - logger.warning(f"Dataset validation failed: {str(exception)}") - return None, error - - except (PydanticValidationError, DatasetNotFoundException): - raise - - except Exception as e: - message = f"Create/update failed for dataset '{dataset.fides_key}'" - logger.warning(f"{message}: {str(e)}") - error = BulkUpdateFailed( - message="Dataset create/update failed.", - data=dataset.model_dump(), - ) - return None, error - - def bulk_create_or_update_datasets( - self, - connection_config: ConnectionConfig, - datasets: Union[List[DatasetConfigCtlDataset], List[FideslangDataset]], - ) -> BulkPutDataset: - """Create or update multiple datasets""" - created_or_updated: List[FideslangDataset] = [] - failed: List[BulkUpdateFailed] = [] - - logger.info("Starting bulk upsert for {} datasets", len(datasets)) - - for item in datasets: - dataset_result, error = self.create_or_update_dataset( - connection_config=connection_config, - dataset=item, - ) - - if dataset_result: - created_or_updated.append(dataset_result) - if error: - failed.append(error) - - return BulkPutDataset( - succeeded=created_or_updated, - failed=failed, - ) - - def clean_datasets(self) -> Tuple[List[str], List[str]]: - datasets = self.db.execute(select([CtlDataset])).scalars().all() - return _run_clean_datasets(self.db, datasets) - def validate_dataset( - self, connection_config: ConnectionConfig, dataset: FideslangDataset + self, + dataset: FideslangDataset, ) -> ValidateDatasetResponse: - - return DatasetValidator(self.db, dataset, connection_config).validate() - - def get_dataset_reachability( - self, dataset_config: DatasetConfig, policy: Optional[Policy] = None - ) -> Tuple[bool, Optional[str]]: """ - Determines if the given dataset is reachable along with an error message + Validates a standalone dataset for create/update operations, performing all necessary validations. """ - # Get all the dataset configs that are not associated with a disabled connection - datasets = DatasetConfig.all(db=self.db) - dataset_graphs = [ - dataset_config.get_graph() - for dataset_config in datasets - if not dataset_config.connection_config.disabled - ] + return DatasetValidator(self.db, dataset).validate() - # We still want to check reachability even if our dataset config's connection is disabled. - # We also consider the siblings, because if the connection is enabled, then all the - # datasets will be enabled with it. - sibling_dataset_graphs = [] - if dataset_config.connection_config.disabled: - sibling_datasets = dataset_config.connection_config.datasets - sibling_dataset_graphs = [ - dataset_config.get_graph() for dataset_config in sibling_datasets - ] + def create_dataset(self, dataset: FideslangDataset) -> CtlDataset: + """Create a new dataset with validation""" - try: - dataset_graph = DatasetGraph(*dataset_graphs, *sibling_dataset_graphs) - except ValidationError as exc: - return ( - False, - f'The following dataset references do not exist "{", ".join(exc.errors)}"', - ) + self.validate_dataset(dataset) + data_dict = dataset.model_dump(mode="json") + return CtlDataset.create(self.db, data=data_dict) - # dummy data is enough to determine traversability - identity_seed: Dict[str, str] = { - k: "something" for k in dataset_graph.identity_keys.values() - } - - try: - Traversal(dataset_graph, identity_seed, policy) - except UnreachableNodesError as exc: - return ( - False, - f'The following collections are not reachable "{", ".join(exc.errors)}"', - ) + def update_dataset(self, dataset: FideslangDataset) -> CtlDataset: + """Update an existing dataset with validation""" - return True, None + self.validate_dataset(dataset) + existing = _get_ctl_dataset(self.db, dataset.fides_key) + if not existing: + raise DatasetNotFoundException(f"Dataset {dataset.fides_key} not found") - def run_test_access_request( - self, - policy: Policy, - dataset_config: DatasetConfig, - input_data: Dict[str, Any], - ) -> PrivacyRequest: - """ - Creates a privacy request with a source of "Dataset test" that runs an access request for a single dataset. - The input data is used to mock any external dataset values referenced by our dataset so that it can run in - complete isolation. - """ + # Update the dataset + data_dict = dataset.model_dump(mode="json") + return existing.update(self.db, data=data_dict) - # Create a privacy request with a source of "Dataset test" - # so it's not shown to the user. - privacy_request = PrivacyRequest.create( - db=self.db, - data={ - "policy_id": policy.id, - "source": PrivacyRequestSource.dataset_test, - "status": PrivacyRequestStatus.in_processing, - }, - ) + def get_dataset(self, fides_key: str) -> CtlDataset: + """Get a single dataset by fides key""" + dataset = _get_ctl_dataset(self.db, fides_key) + if not dataset: + raise DatasetNotFoundException(f"Dataset {fides_key} not found") + return dataset - # Remove periods and colons to avoid them being parsed as path delimiters downstream. - escaped_input_data = { - key.replace(".", "_").replace(":", "_"): value - for key, value in input_data.items() - } + def delete_dataset(self, fides_key: str) -> None: + """Delete a dataset by fides key""" + dataset = self.get_dataset(fides_key) + dataset.delete(self.db) - # Manually cache the input data as identity data. - # We're doing a bit of trickery here to avoid asking for labels for custom identities. - predefined_fields = Identity.model_fields.keys() - input_identity = { - key: ( - value - if key in predefined_fields - else LabeledIdentity(label=key, value=value) - ) - for key, value in escaped_input_data.items() - } - privacy_request.cache_identity(input_identity) - - graph_dataset = dataset_config.get_graph() - modified_graph_dataset = replace_references_with_identities( - dataset_config.fides_key, graph_dataset - ) - - dataset_graph = DatasetGraph(modified_graph_dataset) - connection_config = dataset_config.connection_config + def clean_datasets(self) -> Tuple[List[str], List[str]]: + datasets = self.db.execute(select([CtlDataset])).scalars().all() + return _run_clean_datasets(self.db, datasets) - from fides.api.task.create_request_tasks import run_access_request - # Finally invoke the existing DSR 3.0 access request task - run_access_request( - privacy_request, - policy, - dataset_graph, - [connection_config], - escaped_input_data, - self.db, - privacy_request_proceed=False, +def _get_ctl_dataset(db: Session, fides_key: str) -> CtlDataset: + """Helper to get CTL dataset by fides_key""" + ctl_dataset = db.query(CtlDataset).filter(CtlDataset.fides_key == fides_key).first() + if not ctl_dataset: + raise DatasetNotFoundException( + f"No CTL dataset found with fides_key '{fides_key}'" ) - return privacy_request - - -def get_identities_and_references( - dataset_config: DatasetConfig, -) -> Set[str]: - """ - Returns all identity and dataset references in the dataset. - If a field has multiple references only the first reference will be considered. - """ - - result: Set[str] = set() - dataset: GraphDataset = dataset_config.get_graph() - for collection in dataset.collections: - # Process the identities in the collection - result.update(collection.identities().values()) - for _, field_refs in collection.references().items(): - # Take first reference only, we only care that this collection is reachable, - # how we get there doesn't matter for our current use case - ref, edge_direction = field_refs[0] - if edge_direction == "from" and ref.dataset != dataset_config.fides_key: - result.add(ref.value) - return result - - -def replace_references_with_identities( - dataset_key: str, graph_dataset: GraphDataset -) -> GraphDataset: - """ - Replace external field references with identity values for testing. - - Creates a copy of the graph dataset and replaces dataset references with - equivalent identity references that can be seeded directly. This allows - testing a single dataset in isolation without needing to load data from - referenced external datasets. - """ - - modified_graph_dataset = deepcopy(graph_dataset) - - for collection in modified_graph_dataset.collections: - for field in collection.fields: - for ref, edge_direction in field.references[:]: - if edge_direction == "from" and ref.dataset != dataset_key: - field.identity = f"{ref.dataset}_{ref.collection}_{'_'.join(ref.field_path.levels)}" - field.references.remove((ref, "from")) - - return modified_graph_dataset - - -def validate_data_categories(dataset: FideslangDataset, db: Session) -> None: - """Validate data categories on a given Dataset - - As a separate method because we want to be able to match against data_categories in the - database instead of a static list. - """ - defined_data_categories: List[FidesKey] = get_data_categories_from_db(db) - validate_data_categories_against_db(dataset, defined_data_categories) + return ctl_dataset def _run_clean_datasets( @@ -380,13 +142,3 @@ def _recursive_clean_fields(fields: List[dict]) -> List[dict]: field["fields"] = _recursive_clean_fields(field["fields"]) cleaned_fields.append(field) return cleaned_fields - - -def _get_ctl_dataset(db: Session, fides_key: str) -> CtlDataset: - """Helper to get CTL dataset by fides_key""" - ctl_dataset = db.query(CtlDataset).filter(CtlDataset.fides_key == fides_key).first() - if not ctl_dataset: - raise DatasetNotFoundException( - f"No CTL dataset found with fides_key '{fides_key}'" - ) - return ctl_dataset diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_config_endpoints.py similarity index 99% rename from tests/ops/api/v1/endpoints/test_dataset_endpoints.py rename to tests/ops/api/v1/endpoints/test_dataset_config_endpoints.py index bc97e5508d..54935df524 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_config_endpoints.py @@ -896,7 +896,7 @@ def test_patch_dataset_configs_fides_key_mismatch( ) @mock.patch( - "fides.service.dataset.dataset_service.DatasetConfig.upsert_with_ctl_dataset" + "fides.service.dataset.dataset_config_service.DatasetConfig.upsert_with_ctl_dataset" ) def test_patch_dataset_configs_failed_response( self, @@ -1827,7 +1827,7 @@ def test_patch_datasets_fides_key_mismatch( ) @mock.patch( - "fides.service.dataset.dataset_service.DatasetConfig.upsert_with_ctl_dataset" + "fides.service.dataset.dataset_config_service.DatasetConfig.upsert_with_ctl_dataset" ) def test_patch_datasets_failed_response( self, @@ -1923,7 +1923,7 @@ def test_patch_dataset_invalid_content( assert response.status_code == 400 @mock.patch( - "fides.service.dataset.dataset_service.DatasetConfig.upsert_with_ctl_dataset" + "fides.service.dataset.dataset_config_service.DatasetConfig.upsert_with_ctl_dataset" ) def test_patch_datasets_failed_response( self, diff --git a/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py index 7b6f69e7d7..28eba3a61b 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_test_endpoints.py @@ -15,7 +15,9 @@ DATASET_READ, DATASET_TEST, ) -from tests.ops.api.v1.endpoints.test_dataset_endpoints import get_connection_dataset_url +from tests.ops.api.v1.endpoints.test_dataset_config_endpoints import ( + get_connection_dataset_url, +) class TestDatasetInputs: diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 8effe85c27..7409af842c 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -113,7 +113,9 @@ ) from fides.config import CONFIG from tests.conftest import generate_role_header_for_user -from tests.ops.api.v1.endpoints.test_dataset_endpoints import get_connection_dataset_url +from tests.ops.api.v1.endpoints.test_dataset_config_endpoints import ( + get_connection_dataset_url, +) page_size = Params().size diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index b85393cf81..fbdee71dcf 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -28,7 +28,7 @@ V1_URL_PREFIX, ) from fides.config import CONFIG -from tests.ops.api.v1.endpoints.test_dataset_endpoints import _reject_key +from tests.ops.api.v1.endpoints.test_dataset_config_endpoints import _reject_key from tests.ops.test_helpers.saas_test_utils import create_zip_file diff --git a/tests/ops/service/dataset/test_dataset_service.py b/tests/ops/service/dataset/test_dataset_service.py index e97ec87aa7..6a0983631d 100644 --- a/tests/ops/service/dataset/test_dataset_service.py +++ b/tests/ops/service/dataset/test_dataset_service.py @@ -9,7 +9,7 @@ from fides.api.models.datasetconfig import DatasetConfig from fides.api.models.policy import Policy from fides.api.schemas.policy import ActionType -from fides.service.dataset.dataset_service import ( +from fides.service.dataset.dataset_config_service import ( DatasetService, get_identities_and_references, )