diff --git a/CHANGELOG.md b/CHANGELOG.md index dabee53aa2..e76b4dd0b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) - Adds partitioning, custom identities, multiple identities to test coverage for BigQuery Enterprise [#5618](https://github.com/ethyca/fides/pull/5618) +- Added Datahub groundwork required by Fidesplus [#5666](https://github.com/ethyca/fides/pull/5666) ### Changed - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) diff --git a/clients/admin-ui/src/features/datastore-connections/constants.ts b/clients/admin-ui/src/features/datastore-connections/constants.ts index 54896ffba7..3806ea15e4 100644 --- a/clients/admin-ui/src/features/datastore-connections/constants.ts +++ b/clients/admin-ui/src/features/datastore-connections/constants.ts @@ -36,6 +36,7 @@ export const CONNECTOR_LOGOS_PATH = "/images/connector-logos/"; export const CONNECTION_TYPE_LOGO_MAP = new Map([ [ConnectionType.ATTENTIVE_EMAIL, "attentive.svg"], [ConnectionType.BIGQUERY, "bigquery.svg"], + [ConnectionType.DATAHUB, "datahub.svg"], [ConnectionType.DYNAMODB, "dynamodb.svg"], [ConnectionType.GENERIC_CONSENT_EMAIL, "ethyca.svg"], [ConnectionType.GENERIC_ERASURE_EMAIL, "ethyca.svg"], diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx index 117e78efba..fc6baf0b65 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx @@ -95,39 +95,18 @@ const ConnectionForm = ({ connectionConfig, systemFidesKey }: Props) => { onClose={uploadTemplateModal.onClose} /> - - {selectedConnectionOption?.type === SystemType.DATABASE ? ( - - ) : null} - {selectedConnectionOption?.type === SystemType.SAAS && - selectedConnectionOption ? ( - - ) : null} - {selectedConnectionOption?.type === SystemType.MANUAL && - selectedConnectionOption ? ( + {selectedConnectionOption?.type && + [ + SystemType.DATABASE, + SystemType.DATA_CATALOG, + SystemType.SAAS, + SystemType.MANUAL, + SystemType.EMAIL, + ].includes(selectedConnectionOption.type) ? ( - ) : null} - {selectedConnectionOption?.type === SystemType.EMAIL && - selectedConnectionOption ? ( - ) : null} diff --git a/clients/admin-ui/src/features/integrations/ConnectionCategory.ts b/clients/admin-ui/src/features/integrations/ConnectionCategory.ts index 684f33a0d0..3c0807d2d6 100644 --- a/clients/admin-ui/src/features/integrations/ConnectionCategory.ts +++ b/clients/admin-ui/src/features/integrations/ConnectionCategory.ts @@ -1,4 +1,5 @@ export enum ConnectionCategory { + DATA_CATALOG = "Data Catalog", DATA_WAREHOUSE = "Data Warehouse", DATABASE = "Database", } diff --git a/clients/admin-ui/src/features/integrations/add-integration/SelectIntegrationType.tsx b/clients/admin-ui/src/features/integrations/add-integration/SelectIntegrationType.tsx index a964b7c3b7..2a2e5639b8 100644 --- a/clients/admin-ui/src/features/integrations/add-integration/SelectIntegrationType.tsx +++ b/clients/admin-ui/src/features/integrations/add-integration/SelectIntegrationType.tsx @@ -1,6 +1,7 @@ import { AntButton as Button, Flex, Spacer, TabList, Tabs } from "fidesui"; import { FidesTab } from "~/features/common/DataTabs"; +import { useFlags } from "~/features/common/features"; import FidesSpinner from "~/features/common/FidesSpinner"; import { INTEGRATION_TYPE_LIST, @@ -23,6 +24,10 @@ const SelectIntegrationType = ({ const { tabIndex, onChangeFilter, isFiltering, filteredTypes, tabs } = useIntegrationFilterTabs(INTEGRATION_TYPE_LIST); + const { + flags: { datahub }, + } = useFlags(); + return ( <> @@ -36,16 +41,21 @@ const SelectIntegrationType = ({ ) : ( - {filteredTypes.map((i) => ( - onConfigureClick(i)} - otherButtons={ - - } - /> - ))} + {filteredTypes.map((i) => { + if (!datahub && i.placeholder.connection_type === "datahub") { + return null; + } + return ( + onConfigureClick(i)} + otherButtons={ + + } + /> + ); + })} )} diff --git a/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx b/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx index c2211950d9..468110fab3 100644 --- a/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx +++ b/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx @@ -2,6 +2,7 @@ import { ReactNode } from "react"; import { ConnectionCategory } from "~/features/integrations/ConnectionCategory"; import BIGQUERY_TYPE_INFO from "~/features/integrations/integration-type-info/bigqueryInfo"; +import DATAHUB_TYPE_INFO from "~/features/integrations/integration-type-info/datahubInfo"; import DYNAMO_TYPE_INFO from "~/features/integrations/integration-type-info/dynamoInfo"; import GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLMySQLInfo"; import GOOGLE_CLOUD_SQL_POSTGRES_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLPostgresInfo"; @@ -27,6 +28,7 @@ export type IntegrationTypeInfo = { const INTEGRATION_TYPE_MAP: { [K in ConnectionType]?: IntegrationTypeInfo } = { [ConnectionType.BIGQUERY]: BIGQUERY_TYPE_INFO, + [ConnectionType.DATAHUB]: DATAHUB_TYPE_INFO, [ConnectionType.DYNAMODB]: DYNAMO_TYPE_INFO, [ConnectionType.GOOGLE_CLOUD_SQL_MYSQL]: GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO, [ConnectionType.GOOGLE_CLOUD_SQL_POSTGRES]: diff --git a/clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx b/clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx new file mode 100644 index 0000000000..008af51432 --- /dev/null +++ b/clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx @@ -0,0 +1,52 @@ +import { ListItem } from "fidesui"; + +import { + InfoHeading, + InfoText, + InfoUnorderedList, +} from "~/features/common/copy/components"; +import ShowMoreContent from "~/features/common/copy/ShowMoreContent"; +import { ConnectionCategory } from "~/features/integrations/ConnectionCategory"; +import { AccessLevel, ConnectionType } from "~/types/api"; + +export const DATAHUB_PLACEHOLDER = { + name: "Datahub", + key: "datahub_placeholder", + connection_type: ConnectionType.DATAHUB, + access: AccessLevel.READ, + created_at: "", +}; + +export const DATAHUB_TAGS = ["Data catalog"]; + +export const DatahubOverview = () => ( + <> + + + DataHub is a metadata platform designed to help organizations manage and + govern their data. It acts as a centralized repository for tracking and + discovering data assets across an organization, helping data teams + understand where their data resides, how it's used, and how it flows + through various systems. + + + + + Data Catalog + + + + Placeholder + + + +); + +const DATAHUB_TYPE_INFO = { + placeholder: DATAHUB_PLACEHOLDER, + category: ConnectionCategory.DATA_CATALOG, + overview: , + tags: DATAHUB_TAGS, +}; + +export default DATAHUB_TYPE_INFO; diff --git a/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx b/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx index 9ed403fb5c..2cfe7c2493 100644 --- a/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx +++ b/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx @@ -5,6 +5,7 @@ import { IntegrationTypeInfo } from "~/features/integrations/add-integration/all export enum IntegrationFilterTabs { ALL = "All", DATABASE = "Database", + DATA_CATALOG = "Data Catalog", DATA_WAREHOUSE = "Data Warehouse", } diff --git a/clients/admin-ui/src/flags.json b/clients/admin-ui/src/flags.json index 5f2b32cea2..1af54fd494 100644 --- a/clients/admin-ui/src/flags.json +++ b/clients/admin-ui/src/flags.json @@ -53,5 +53,11 @@ "development": true, "test": true, "production": false + }, + "datahub": { + "description": "Share Fides data categories with your Datahub instance", + "development": true, + "test": true, + "production": false } } diff --git a/clients/admin-ui/src/types/api/models/SystemType.ts b/clients/admin-ui/src/types/api/models/SystemType.ts index b08e9d168e..3d1938b963 100644 --- a/clients/admin-ui/src/types/api/models/SystemType.ts +++ b/clients/admin-ui/src/types/api/models/SystemType.ts @@ -3,8 +3,9 @@ /* eslint-disable */ export enum SystemType { - SAAS = "saas", + DATA_CATALOG = "data_catalog", DATABASE = "database", - MANUAL = "manual", EMAIL = "email", + MANUAL = "manual", + SAAS = "saas", } diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 2dc518cd44..6775396734 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -2,7 +2,7 @@ import enum from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type +from typing import TYPE_CHECKING, Any, List, Optional, Type from loguru import logger from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event @@ -24,6 +24,7 @@ if TYPE_CHECKING: from fides.api.models.detection_discovery import MonitorConfig + from fides.api.schemas.connection_configuration.enums.system_type import SystemType class ConnectionTestStatus(enum.Enum): @@ -72,7 +73,7 @@ def human_readable(self) -> str: """Human-readable mapping for ConnectionTypes Add to this mapping if you add a new ConnectionType """ - readable_mapping: Dict[str, str] = { + readable_mapping: dict[str, str] = { ConnectionType.attentive_email.value: "Attentive Email", ConnectionType.bigquery.value: "BigQuery", ConnectionType.datahub.value: "DataHub", @@ -108,6 +109,47 @@ def human_readable(self) -> str: "Add new ConnectionType to human_readable mapping" ) + @property + def system_type(self) -> "SystemType": + from fides.api.schemas.connection_configuration.enums.system_type import ( + SystemType, + ) + + system_type_mapping: dict[str, SystemType] = { + ConnectionType.attentive_email.value: SystemType.email, + ConnectionType.bigquery.value: SystemType.database, + ConnectionType.datahub.value: SystemType.data_catalog, + ConnectionType.dynamic_erasure_email.value: SystemType.email, + ConnectionType.dynamodb.value: SystemType.database, + ConnectionType.fides.value: SystemType.manual, + ConnectionType.generic_consent_email.value: SystemType.email, + ConnectionType.generic_erasure_email.value: SystemType.email, + ConnectionType.google_cloud_sql_mysql.value: SystemType.database, + ConnectionType.google_cloud_sql_postgres.value: SystemType.database, + ConnectionType.https.value: SystemType.manual, + ConnectionType.manual_webhook.value: SystemType.manual, + ConnectionType.manual.value: SystemType.manual, + ConnectionType.mariadb.value: SystemType.database, + ConnectionType.mongodb.value: SystemType.database, + ConnectionType.mssql.value: SystemType.database, + ConnectionType.mysql.value: SystemType.database, + ConnectionType.postgres.value: SystemType.database, + ConnectionType.rds_mysql.value: SystemType.database, + ConnectionType.rds_postgres.value: SystemType.database, + ConnectionType.redshift.value: SystemType.database, + ConnectionType.s3.value: SystemType.database, + ConnectionType.saas.value: SystemType.saas, + ConnectionType.scylla.value: SystemType.database, + ConnectionType.snowflake.value: SystemType.database, + ConnectionType.sovrn.value: SystemType.email, + ConnectionType.timescale.value: SystemType.database, + } + + try: + return system_type_mapping[self.value] + except KeyError: + raise NotImplementedError("Add new ConnectionType to system_type mapping") + class AccessLevel(enum.Enum): """ diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py index bfda71ed50..bde79ad24f 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py @@ -1,4 +1,3 @@ -from enum import Enum from typing import ClassVar, List from pydantic import Field @@ -10,14 +9,6 @@ ) -class PeriodicIntegrationFrequency(Enum): - """Enum for periodic integration frequency""" - - daily = "daily" - weekly = "weekly" - monthly = "monthly" - - class DatahubSchema(ConnectionConfigSecretsSchema): datahub_server_url: AnyHttpUrlStringRemovesSlash = Field( title="DataHub Server URL", @@ -28,11 +19,6 @@ class DatahubSchema(ConnectionConfigSecretsSchema): description="The token used to authenticate with your DataHub server.", json_schema_extra={"sensitive": True}, ) - frequency: PeriodicIntegrationFrequency = Field( - title="Frequency", - description="The frequency at which the integration should run. Defaults to daily.", - default=PeriodicIntegrationFrequency.daily, - ) _required_components: ClassVar[List[str]] = ["datahub_server_url", "datahub_token"] diff --git a/src/fides/api/schemas/connection_configuration/enums/system_type.py b/src/fides/api/schemas/connection_configuration/enums/system_type.py index 275a63f3f5..dad517822e 100644 --- a/src/fides/api/schemas/connection_configuration/enums/system_type.py +++ b/src/fides/api/schemas/connection_configuration/enums/system_type.py @@ -2,7 +2,8 @@ class SystemType(Enum): - saas = "saas" + data_catalog = "data_catalog" database = "database" - manual = "manual" email = "email" + manual = "manual" + saas = "saas" diff --git a/src/fides/api/util/connection_type.py b/src/fides/api/util/connection_type.py index 5edae3135a..14c2041d12 100644 --- a/src/fides/api/util/connection_type.py +++ b/src/fides/api/util/connection_type.py @@ -220,9 +220,9 @@ def saas_request_type_filter(connection_type: str) -> bool: if (system_type == SystemType.database or system_type is None) and ( ActionType.access in action_types or ActionType.erasure in action_types ): - database_types: list[str] = sorted( + database_types: list[ConnectionType] = sorted( [ - conn_type.value + conn_type for conn_type in ConnectionType if conn_type not in [ @@ -238,14 +238,15 @@ def saas_request_type_filter(connection_type: str) -> bool: ConnectionType.sovrn, ] and is_match(conn_type.value) - ] + ], + key=lambda x: x.value, ) connection_system_types.extend( [ ConnectionSystemTypeMap( identifier=item, - type=SystemType.database, - human_readable=ConnectionType(item).human_readable, + type=item.system_type, + human_readable=item.human_readable, supported_actions=[ActionType.access, ActionType.erasure], ) for item in database_types @@ -336,4 +337,5 @@ def saas_request_type_filter(connection_type: str) -> bool: for email_type in email_types ] ) + return connection_system_types diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 83fade38c0..3bffd68796 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -2018,7 +2018,6 @@ def test_put_datahub_connection_config_secrets( payload = { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", - "frequency": "weekly", } resp = api_client.put( url + "?verify=False", @@ -2035,7 +2034,6 @@ def test_put_datahub_connection_config_secrets( assert datahub_connection_config_no_secrets.secrets == { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", - "frequency": "weekly", } assert datahub_connection_config_no_secrets.last_test_timestamp is None assert datahub_connection_config_no_secrets.last_test_succeeded is None @@ -2071,7 +2069,6 @@ def test_put_datahub_connection_config_secrets_default_frequency( assert datahub_connection_config_no_secrets.secrets == { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", - "frequency": "daily", } assert datahub_connection_config_no_secrets.last_test_timestamp is None assert datahub_connection_config_no_secrets.last_test_succeeded is None @@ -2088,7 +2085,7 @@ def test_put_datahub_connection_config_secrets_missing_url( """ url = f"{V1_URL_PREFIX}{CONNECTIONS}/{datahub_connection_config_no_secrets.key}/secret" auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = {"datahub_token": "test", "frequency": "weekly"} + payload = {"datahub_token": "test"} resp = api_client.put( url + "?verify=False", headers=auth_header, @@ -2114,7 +2111,6 @@ def test_put_datahub_connection_config_secrets_missing_token( auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) payload = { "datahub_server_url": "https://datahub.example.com", - "frequency": "weekly", } resp = api_client.put( url + "?verify=False", diff --git a/tests/ops/models/test_connectionconfig.py b/tests/ops/models/test_connectionconfig.py index 93978ab8fc..b24a43dbf3 100644 --- a/tests/ops/models/test_connectionconfig.py +++ b/tests/ops/models/test_connectionconfig.py @@ -146,8 +146,16 @@ def test_connection_type_human_readable(self): connection.human_readable # Makes sure all ConnectionTypes have been added to human_readable mapping def test_connection_type_human_readable_invalid(self): - with pytest.raises(ValueError): - ConnectionType("nonmapped_type").human_readable() + with pytest.raises(NotImplementedError): + ConnectionType("nonmapped_type").human_readable + + def test_connection_type_system_type(self): + for connection in ConnectionType: + connection.system_type # Makes sure all ConnectionTypes have been added to system_type mapping + + def test_connection_type_system_type_invalid(self): + with pytest.raises(NotImplementedError): + ConnectionType("nonmapped_type").system_type def test_system_key(self, db, connection_config, system): assert connection_config.system_key == connection_config.name