HJ 181 - Datahub groundwork and UI

CHANGELOG.md

@@ -25,6 +25,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o
 - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622)
 - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629)
 - Adds partitioning, custom identities, multiple identities to test coverage for BigQuery Enterprise [#5618](https://github.com/ethyca/fides/pull/5618)
+- Added Datahub groundwork required by Fidesplus [#5666](https://github.com/ethyca/fides/pull/5666)
 
 ### Changed
 - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656)

clients/admin-ui/src/features/datastore-connections/constants.ts

@@ -36,6 +36,7 @@ export const CONNECTOR_LOGOS_PATH = "/images/connector-logos/";
 export const CONNECTION_TYPE_LOGO_MAP = new Map<ConnectionType, string>([
   [ConnectionType.ATTENTIVE_EMAIL, "attentive.svg"],
   [ConnectionType.BIGQUERY, "bigquery.svg"],
+  [ConnectionType.DATAHUB, "datahub.svg"],
   [ConnectionType.DYNAMODB, "dynamodb.svg"],
   [ConnectionType.GENERIC_CONSENT_EMAIL, "ethyca.svg"],
   [ConnectionType.GENERIC_ERASURE_EMAIL, "ethyca.svg"],

clients/admin-ui/src/features/datastore-connections/system_portal_config/ConnectionForm.tsx

@@ -95,39 +95,18 @@ const ConnectionForm = ({ connectionConfig, systemFidesKey }: Props) => {
           onClose={uploadTemplateModal.onClose}
         />
       </Flex>
-
-      {selectedConnectionOption?.type === SystemType.DATABASE ? (
-        <ConnectorParameters
-          connectionConfig={connectionConfig}
-          setSelectedConnectionOption={setSelectedConnectionOption}
-          connectionOption={selectedConnectionOption}
-          systemFidesKey={systemFidesKey}
-        />
-      ) : null}
-      {selectedConnectionOption?.type === SystemType.SAAS &&
-      selectedConnectionOption ? (
-        <ConnectorParameters
-          connectionOption={selectedConnectionOption}
-          setSelectedConnectionOption={setSelectedConnectionOption}
-          connectionConfig={connectionConfig}
-          systemFidesKey={systemFidesKey}
-        />
-      ) : null}
-      {selectedConnectionOption?.type === SystemType.MANUAL &&
-      selectedConnectionOption ? (
+      {selectedConnectionOption?.type &&
+      [
+        SystemType.DATABASE,
+        SystemType.DATA_CATALOG,
+        SystemType.SAAS,
+        SystemType.MANUAL,
+        SystemType.EMAIL,
+      ].includes(selectedConnectionOption.type) ? (
         <ConnectorParameters
-          connectionOption={selectedConnectionOption}
-          setSelectedConnectionOption={setSelectedConnectionOption}
           connectionConfig={connectionConfig}
-          systemFidesKey={systemFidesKey}
-        />
-      ) : null}
-      {selectedConnectionOption?.type === SystemType.EMAIL &&
-      selectedConnectionOption ? (
-        <ConnectorParameters
           connectionOption={selectedConnectionOption}
           setSelectedConnectionOption={setSelectedConnectionOption}
-          connectionConfig={connectionConfig}
           systemFidesKey={systemFidesKey}
         />
       ) : null}

clients/admin-ui/src/features/integrations/ConnectionCategory.ts

@@ -1,4 +1,5 @@
 export enum ConnectionCategory {
+  DATA_CATALOG = "Data Catalog",
   DATA_WAREHOUSE = "Data Warehouse",
   DATABASE = "Database",
 }

clients/admin-ui/src/features/integrations/add-integration/SelectIntegrationType.tsx

@@ -1,6 +1,7 @@
 import { AntButton as Button, Flex, Spacer, TabList, Tabs } from "fidesui";
 
 import { FidesTab } from "~/features/common/DataTabs";
+import { useFlags } from "~/features/common/features";
 import FidesSpinner from "~/features/common/FidesSpinner";
 import {
   INTEGRATION_TYPE_LIST,
@@ -23,6 +24,10 @@ const SelectIntegrationType = ({
   const { tabIndex, onChangeFilter, isFiltering, filteredTypes, tabs } =
     useIntegrationFilterTabs(INTEGRATION_TYPE_LIST);
 
+  const {
+    flags: { datahub },
+  } = useFlags();
+
   return (
     <>
       <Tabs index={tabIndex} onChange={onChangeFilter} mb={4}>
@@ -36,16 +41,21 @@ const SelectIntegrationType = ({
         <FidesSpinner />
       ) : (
         <Flex direction="column">
-          {filteredTypes.map((i) => (
-            <IntegrationBox
-              integration={i.placeholder}
-              key={i.placeholder.key}
-              onConfigureClick={() => onConfigureClick(i)}
-              otherButtons={
-                <Button onClick={() => onDetailClick(i)}>Details</Button>
-              }
-            />
-          ))}
+          {filteredTypes.map((i) => {
+            if (!datahub && i.placeholder.connection_type === "datahub") {
+              return null;
+            }
+            return (
+              <IntegrationBox
+                integration={i.placeholder}
+                key={i.placeholder.key}
+                onConfigureClick={() => onConfigureClick(i)}
+                otherButtons={
+                  <Button onClick={() => onDetailClick(i)}>Details</Button>
+                }
+              />
+            );
+          })}
         </Flex>
       )}
       <Flex>

clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx

@@ -2,6 +2,7 @@ import { ReactNode } from "react";
 
 import { ConnectionCategory } from "~/features/integrations/ConnectionCategory";
 import BIGQUERY_TYPE_INFO from "~/features/integrations/integration-type-info/bigqueryInfo";
+import DATAHUB_TYPE_INFO from "~/features/integrations/integration-type-info/datahubInfo";
 import DYNAMO_TYPE_INFO from "~/features/integrations/integration-type-info/dynamoInfo";
 import GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLMySQLInfo";
 import GOOGLE_CLOUD_SQL_POSTGRES_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLPostgresInfo";
@@ -27,6 +28,7 @@ export type IntegrationTypeInfo = {
 
 const INTEGRATION_TYPE_MAP: { [K in ConnectionType]?: IntegrationTypeInfo } = {
   [ConnectionType.BIGQUERY]: BIGQUERY_TYPE_INFO,
+  [ConnectionType.DATAHUB]: DATAHUB_TYPE_INFO,
   [ConnectionType.DYNAMODB]: DYNAMO_TYPE_INFO,
   [ConnectionType.GOOGLE_CLOUD_SQL_MYSQL]: GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO,
   [ConnectionType.GOOGLE_CLOUD_SQL_POSTGRES]:

clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx

@@ -0,0 +1,52 @@
+import { ListItem } from "fidesui";
+
+import {
+  InfoHeading,
+  InfoText,
+  InfoUnorderedList,
+} from "~/features/common/copy/components";
+import ShowMoreContent from "~/features/common/copy/ShowMoreContent";
+import { ConnectionCategory } from "~/features/integrations/ConnectionCategory";
+import { AccessLevel, ConnectionType } from "~/types/api";
+
+export const DATAHUB_PLACEHOLDER = {
+  name: "Datahub",
+  key: "datahub_placeholder",
+  connection_type: ConnectionType.DATAHUB,
+  access: AccessLevel.READ,
+  created_at: "",
+};
+
+export const DATAHUB_TAGS = ["Data catalog"];
+
+export const DatahubOverview = () => (
+  <>
+    <InfoHeading text="Overview" />
+    <InfoText>
+      DataHub is a metadata platform designed to help organizations manage and
+      govern their data. It acts as a centralized repository for tracking and
+      discovering data assets across an organization, helping data teams
+      understand where their data resides, how it&apos;s used, and how it flows
+      through various systems.
+    </InfoText>
+    <ShowMoreContent>
+      <InfoHeading text="Categories" />
+      <InfoUnorderedList>
+        <ListItem>Data Catalog</ListItem>
+      </InfoUnorderedList>
+      <InfoHeading text="Permissions" />
+      <InfoUnorderedList>
+        <ListItem>Placeholder</ListItem>
+      </InfoUnorderedList>
+    </ShowMoreContent>
+  </>
+);
+
+const DATAHUB_TYPE_INFO = {
+  placeholder: DATAHUB_PLACEHOLDER,
+  category: ConnectionCategory.DATA_CATALOG,
+  overview: <DatahubOverview />,
+  tags: DATAHUB_TAGS,
+};
+
+export default DATAHUB_TYPE_INFO;

clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx

@@ -5,6 +5,7 @@ import { IntegrationTypeInfo } from "~/features/integrations/add-integration/all
 export enum IntegrationFilterTabs {
   ALL = "All",
   DATABASE = "Database",
+  DATA_CATALOG = "Data Catalog",
   DATA_WAREHOUSE = "Data Warehouse",
 }
 

clients/admin-ui/src/flags.json

@@ -53,5 +53,11 @@
     "development": true,
     "test": true,
     "production": false
+  },
+  "datahub": {
+    "description": "Share Fides data categories with your Datahub instance",
+    "development": true,
+    "test": true,
+    "production": false
   }
 }

clients/admin-ui/src/types/api/models/SystemType.ts

@@ -3,8 +3,9 @@
 /* eslint-disable */
 
 export enum SystemType {
-  SAAS = "saas",
+  DATA_CATALOG = "data_catalog",
   DATABASE = "database",
-  MANUAL = "manual",
   EMAIL = "email",
+  MANUAL = "manual",
+  SAAS = "saas",
 }

src/fides/api/models/connectionconfig.py

@@ -2,7 +2,7 @@
 
 import enum
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type
+from typing import TYPE_CHECKING, Any, List, Optional, Type
 
 from loguru import logger
 from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event
@@ -24,6 +24,7 @@
 
 if TYPE_CHECKING:
     from fides.api.models.detection_discovery import MonitorConfig
+    from fides.api.schemas.connection_configuration.enums.system_type import SystemType
 
 
 class ConnectionTestStatus(enum.Enum):
@@ -72,7 +73,7 @@ def human_readable(self) -> str:
         """Human-readable mapping for ConnectionTypes
         Add to this mapping if you add a new ConnectionType
         """
-        readable_mapping: Dict[str, str] = {
+        readable_mapping: dict[str, str] = {
             ConnectionType.attentive_email.value: "Attentive Email",
             ConnectionType.bigquery.value: "BigQuery",
             ConnectionType.datahub.value: "DataHub",
@@ -108,6 +109,49 @@ def human_readable(self) -> str:
                 "Add new ConnectionType to human_readable mapping"
             )
 
+    @property
+    def system_type(self) -> "SystemType":
+        from fides.api.schemas.connection_configuration.enums.system_type import (
+            SystemType,
+        )
+
+        system_type_mapping: dict[str, SystemType] = {
+            ConnectionType.attentive_email.value: SystemType.email,
+            ConnectionType.bigquery.value: SystemType.database,
+            ConnectionType.datahub.value: SystemType.data_catalog,
+            ConnectionType.dynamic_erasure_email.value: SystemType.email,
+            ConnectionType.dynamodb.value: SystemType.database,
+            ConnectionType.fides.value: SystemType.manual,
+            ConnectionType.generic_consent_email.value: SystemType.email,
+            ConnectionType.generic_erasure_email.value: SystemType.email,
+            ConnectionType.google_cloud_sql_mysql.value: SystemType.database,
+            ConnectionType.google_cloud_sql_postgres.value: SystemType.database,
+            ConnectionType.https.value: SystemType.manual,
+            ConnectionType.manual_webhook.value: SystemType.manual,
+            ConnectionType.manual.value: SystemType.manual,
+            ConnectionType.mariadb.value: SystemType.database,
+            ConnectionType.mongodb.value: SystemType.database,
+            ConnectionType.mssql.value: SystemType.database,
+            ConnectionType.mysql.value: SystemType.database,
+            ConnectionType.postgres.value: SystemType.database,
+            ConnectionType.rds_mysql.value: SystemType.database,
+            ConnectionType.rds_postgres.value: SystemType.database,
+            ConnectionType.redshift.value: SystemType.database,
+            ConnectionType.s3.value: SystemType.database,
+            ConnectionType.saas.value: SystemType.saas,
+            ConnectionType.scylla.value: SystemType.database,
+            ConnectionType.snowflake.value: SystemType.database,
+            ConnectionType.sovrn.value: SystemType.email,
+            ConnectionType.timescale.value: SystemType.database,
+        }
+
+        try:
+            return system_type_mapping[self.value]
+        except KeyError:
+            raise NotImplementedError(
+                "Add new ConnectionType to system_type mapping"
+            )
+
 
 class AccessLevel(enum.Enum):
     """

src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import ClassVar, List
 
 from pydantic import Field
@@ -10,14 +9,6 @@
 )
 
 
-class PeriodicIntegrationFrequency(Enum):
-    """Enum for periodic integration frequency"""
-
-    daily = "daily"
-    weekly = "weekly"
-    monthly = "monthly"
-
-
 class DatahubSchema(ConnectionConfigSecretsSchema):
     datahub_server_url: AnyHttpUrlStringRemovesSlash = Field(
         title="DataHub Server URL",
@@ -28,11 +19,6 @@ class DatahubSchema(ConnectionConfigSecretsSchema):
         description="The token used to authenticate with your DataHub server.",
         json_schema_extra={"sensitive": True},
     )
-    frequency: PeriodicIntegrationFrequency = Field(
-        title="Frequency",
-        description="The frequency at which the integration should run. Defaults to daily.",
-        default=PeriodicIntegrationFrequency.daily,
-    )
 
     _required_components: ClassVar[List[str]] = ["datahub_server_url", "datahub_token"]
 

src/fides/api/schemas/connection_configuration/enums/system_type.py

@@ -2,7 +2,8 @@
 
 
 class SystemType(Enum):
-    saas = "saas"
+    data_catalog = "data_catalog"
     database = "database"
-    manual = "manual"
     email = "email"
+    manual = "manual"
+    saas = "saas"

src/fides/api/util/connection_type.py

@@ -220,9 +220,9 @@ def saas_request_type_filter(connection_type: str) -> bool:
     if (system_type == SystemType.database or system_type is None) and (
         ActionType.access in action_types or ActionType.erasure in action_types
     ):
-        database_types: list[str] = sorted(
+        database_types: list[ConnectionType] = sorted(
             [
-                conn_type.value
+                conn_type
                 for conn_type in ConnectionType
                 if conn_type
                 not in [
@@ -238,14 +238,15 @@ def saas_request_type_filter(connection_type: str) -> bool:
                     ConnectionType.sovrn,
                 ]
                 and is_match(conn_type.value)
-            ]
+            ],
+            key=lambda x: x.value,
         )
         connection_system_types.extend(
             [
                 ConnectionSystemTypeMap(
                     identifier=item,
-                    type=SystemType.database,
-                    human_readable=ConnectionType(item).human_readable,
+                    type=item.system_type,
+                    human_readable=item.human_readable,
                     supported_actions=[ActionType.access, ActionType.erasure],
                 )
                 for item in database_types
@@ -336,4 +337,5 @@ def saas_request_type_filter(connection_type: str) -> bool:
                 for email_type in email_types
             ]
         )
+
     return connection_system_types