diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1b34dd48..9280d5721b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,12 +38,13 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Fixed - Updating mongodb connectors so it can support usernames and password with URL encoded characters [#5682](https://github.com/ethyca/fides/pull/5682) - After creating a new system, the url is now updated correctly to the new system edit page [#5701](https://github.com/ethyca/fides/pull/5701) - +- Visual fixes for table header buttons [#5693](https://github.com/ethyca/fides/pull/5693) ## [2.53.0](https://github.com/ethyca/fides/compare/2.52.0...2.53.0) ### Added - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) +- Added Data Catalog MVP behind new feature flag [#5628](https://github.com/ethyca/fides/pull/5628) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) - Adds partitioning, custom identities, multiple identities to test coverage for BigQuery Enterprise [#5618](https://github.com/ethyca/fides/pull/5618) - Added Datahub groundwork required by Fidesplus [#5666](https://github.com/ethyca/fides/pull/5666) @@ -211,6 +212,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Fixed - API router sanitizer being too aggressive with NextJS Catch-all Segments [#5438](https://github.com/ethyca/fides/pull/5438) + - Fix rendering of subfield names in D&D tables [#5439](https://github.com/ethyca/fides/pull/5439) - Fix BigQuery `partitioning` queries to properly support multiple identity clauses [#5432](https://github.com/ethyca/fides/pull/5432) ## [2.48.0](https://github.com/ethyca/fides/compare/2.47.1...2.48.0) diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts index 3cc201c343..bc693bffa1 100644 --- a/clients/admin-ui/cypress/e2e/action-center.cy.ts +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -1,4 +1,9 @@ -import { stubActionCenter, stubPlus } from "cypress/support/stubs"; +import { + stubActionCenter, + stubPlus, + stubSystemVendors, + stubVendorList, +} from "cypress/support/stubs"; import { ACTION_CENTER_ROUTE, @@ -119,6 +124,7 @@ describe("Action center", () => { beforeEach(() => { cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}`); cy.wait("@getSystemAggregateResults"); + cy.getByTestId("page-breadcrumb").should("contain", webMonitorKey); // little hack to make sure the webMonitorKey is available before proceeding }); it("should display a breadcrumb", () => { cy.getByTestId("page-breadcrumb").within(() => { @@ -137,12 +143,14 @@ describe("Action center", () => { // cy.getByTestId("column-data_use").should("exist"); cy.getByTestId("column-locations").should("exist"); cy.getByTestId("column-domains").should("exist"); - // TODO: [HJ-343] uncomment when actions column is implemented - // cy.getByTestId("column-actions").should("exist"); + cy.getByTestId("column-actions").should("exist"); cy.getByTestId("row-0-col-system_name").within(() => { - cy.getByTestId("change-icon").should("exist"); // new result + cy.getByTestId("change-icon").should("exist"); cy.contains("Uncategorized assets").should("exist"); }); + cy.getByTestId("row-3-col-system_name").within(() => { + cy.getByTestId("change-icon").should("exist"); // new system + }); // TODO: [HJ-356] uncomment when data use column is implemented /* // data use column should be empty for uncategorized assets cy.getByTestId("row-0-col-data_use").children().should("have.length", 0); @@ -165,6 +173,39 @@ describe("Action center", () => { "contain", "analytics.google.com", ); + cy.getByTestId("row-0-col-actions").within(() => { + cy.getByTestId("add-btn").should("be.disabled"); + }); + }); + it("should ignore all assets in an uncategorized system", () => { + cy.getByTestId("row-0-col-actions").within(() => { + cy.getByTestId("ignore-btn").click({ force: true }); + }); + cy.wait("@ignoreMonitorResultUncategorizedSystem"); + cy.getByTestId("success-alert").should( + "contain", + "108 uncategorized assets have been ignored and will not appear in future scans.", + ); + }); + it("should add all assets in a categorized system", () => { + cy.getByTestId("row-1-col-actions").within(() => { + cy.getByTestId("add-btn").click({ force: true }); + }); + cy.wait("@addMonitorResultSystem"); + cy.getByTestId("success-alert").should( + "contain", + "10 assets from Google Tag Manager have been added to the system inventory.", + ); + }); + it("should ignore all assets in a categorized system", () => { + cy.getByTestId("row-1-col-actions").within(() => { + cy.getByTestId("ignore-btn").click({ force: true }); + }); + cy.wait("@ignoreMonitorResultSystem"); + cy.getByTestId("success-alert").should( + "contain", + "10 assets from Google Tag Manager have been ignored and will not appear in future scans.", + ); }); it("should navigate to table view on row click", () => { cy.getByTestId("row-1").click(); @@ -179,18 +220,70 @@ describe("Action center", () => { }); }); - describe("Action center system assets results", () => { + describe("Action center assets uncategorized results", () => { + const webMonitorKey = "my_web_monitor_1"; + const systemId = "[undefined]"; + beforeEach(() => { + cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}/${systemId}`); + cy.wait("@getSystemAssetsUncategorized"); + cy.getByTestId("page-breadcrumb").should("contain", "Uncategorized"); + }); + it("should render uncategorized asset results view", () => { + cy.getByTestId("search-bar").should("exist"); + cy.getByTestId("pagination-btn").should("exist"); + cy.getByTestId("bulk-actions-menu").should("be.disabled"); + cy.getByTestId("add-all").should("be.disabled"); + + // table columns + cy.getByTestId("column-select").should("exist"); + cy.getByTestId("column-name").should("exist"); + cy.getByTestId("column-resource_type").should("exist"); + cy.getByTestId("column-system").should("exist"); + // TODO: [HJ-369] uncomment when data use column is implemented + // cy.getByTestId("column-data_use").should("exist"); + cy.getByTestId("column-locations").should("exist"); + cy.getByTestId("column-domain").should("exist"); + // TODO: [HJ-344] uncomment when Discovery column is implemented + /* cy.getByTestId("column-with_consent").should("exist"); + cy.getByTestId("row-4-col-with_consent") + .contains("Without consent") + .realHover(); + cy.get(".ant-tooltip-inner").should("contain", "January"); */ + cy.getByTestId("column-actions").should("exist"); + cy.getByTestId("row-0-col-actions").within(() => { + cy.getByTestId("add-btn").should("be.disabled"); + cy.getByTestId("ignore-btn").should("exist"); + }); + }); + it("should allow adding a system on uncategorized assets", () => { + cy.getByTestId("row-0-col-system").within(() => { + cy.getByTestId("add-system-btn").click(); + }); + cy.wait("@getSystemsPaginated"); + cy.getByTestId("system-select").antSelect("Fidesctl System"); + cy.wait("@setAssetSystem"); + cy.getByTestId("system-select").should("not.exist"); + cy.getByTestId("success-alert").should( + "contain", + 'Browser Request "0d22c925-3a81-4f10-bfdc-69a5d67e93bc" has been assigned to Fidesctl System.', + ); + }); + }); + + describe("Action center assets categorized results", () => { const webMonitorKey = "my_web_monitor_1"; const systemId = "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88"; beforeEach(() => { cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}/${systemId}`); cy.wait("@getSystemAssetResults"); + cy.getByTestId("page-breadcrumb").should("contain", systemId); // little hack to make sure the systemId is available before proceeding }); it("should render asset results view", () => { cy.getByTestId("page-breadcrumb").should("contain", systemId); cy.getByTestId("search-bar").should("exist"); cy.getByTestId("pagination-btn").should("exist"); cy.getByTestId("bulk-actions-menu").should("be.disabled"); + cy.getByTestId("add-all").should("exist"); // table columns cy.getByTestId("column-select").should("exist"); @@ -213,11 +306,7 @@ describe("Action center", () => { cy.getByTestId("ignore-btn").should("exist"); }); }); - it.skip("should allow adding a system on uncategorized assets", () => { - // TODO: uncategorized assets are not yet available for testing - }); it("should allow editing a system on categorized assets", () => { - cy.getByTestId("page-breadcrumb").should("contain", systemId); // little hack to make sure the systemId is available before proceeding cy.getByTestId("row-3-col-system").within(() => { cy.getByTestId("system-badge").click(); }); @@ -225,6 +314,10 @@ describe("Action center", () => { cy.getByTestId("system-select").antSelect("Fidesctl System"); cy.wait("@setAssetSystem"); cy.getByTestId("system-select").should("not.exist"); + cy.getByTestId("success-alert").should( + "contain", + 'Browser Request "destination" has been assigned to Fidesctl System.', + ); // Wait for previous UI animations to reset or Cypress chokes on the next part // eslint-disable-next-line cypress/no-unnecessary-waiting @@ -243,20 +336,50 @@ describe("Action center", () => { cy.wait("@setAssetSystem"); cy.getByTestId("success-alert").should("exist"); cy.getByTestId("system-select").should("not.exist"); + cy.getByTestId("success-alert").should( + "contain", + 'Browser Request "collect" has been assigned to Demo Marketing System.', + ); + }); + it("should allow creating a new system and assigning an asset to it", () => { + stubVendorList(); + stubSystemVendors(); + cy.getByTestId("row-4-col-system").within(() => { + cy.getByTestId("system-badge").click(); + }); + cy.wait("@getSystemsPaginated"); + cy.getByTestId("add-new-system").click(); + cy.getByTestId("add-modal-content").should("exist"); + cy.getByTestId("vendor-name-select").antSelect("Aniview LTD"); + cy.getByTestId("save-btn").click(); + // adds new system + cy.wait("@postSystemVendors"); + // assigns asset to new system + cy.wait("@setAssetSystem"); + cy.getByTestId("success-alert").should( + "contain", + 'Test System has been added to your system inventory and the Browser Request "gtm.js" has been assigned to that system.', + ); }); it("should add individual assets", () => { cy.getByTestId("row-0-col-actions").within(() => { cy.getByTestId("add-btn").click({ force: true }); }); cy.wait("@addAssets"); - cy.getByTestId("success-alert").should("exist"); + cy.getByTestId("success-alert").should( + "contain", + 'Browser Request "11020051272" has been added to the system inventory.', + ); }); it("should ignore individual assets", () => { cy.getByTestId("row-0-col-actions").within(() => { cy.getByTestId("ignore-btn").click({ force: true }); }); cy.wait("@ignoreAssets"); - cy.getByTestId("success-alert").should("exist"); + cy.getByTestId("success-alert").should( + "contain", + 'Browser Request "11020051272" has been ignored and will not appear in future scans.', + ); }); it("should bulk add assets", () => { cy.getByTestId("bulk-actions-menu").should("be.disabled"); @@ -268,7 +391,10 @@ describe("Action center", () => { cy.getByTestId("bulk-actions-menu").click(); cy.getByTestId("bulk-add").click(); cy.wait("@addAssets"); - cy.getByTestId("success-alert").should("exist"); + cy.getByTestId("success-alert").should( + "contain", + "3 assets from Google Tag Manager have been added to the system inventory.", + ); }); it("should bulk ignore assets", () => { cy.getByTestId("bulk-actions-menu").should("be.disabled"); @@ -280,15 +406,29 @@ describe("Action center", () => { cy.getByTestId("bulk-actions-menu").click(); cy.getByTestId("bulk-ignore").click(); cy.wait("@ignoreAssets"); - cy.getByTestId("success-alert").should("exist"); + cy.getByTestId("success-alert").should( + "contain", + "3 assets from Google Tag Manager have been ignored and will not appear in future scans.", + ); }); - it.skip("should add all assets", () => { - // TODO: [HJ-343] unskip when add all is implemented + it("should add all assets", () => { + cy.intercept( + "POST", + "/api/v1/plus/discovery-monitor/*/promote*", + (req) => { + req.on("response", (res) => { + res.setDelay(100); // slight delay allows us to check for the loading state below + }); + }, + ).as("slowRequest"); cy.getByTestId("add-all").click(); cy.getByTestId("add-all").should("have.class", "ant-btn-loading"); - cy.wait("@addAssets"); + cy.wait("@slowRequest"); cy.url().should("not.contain", systemId); - cy.getByTestId("success-alert").should("exist"); + cy.getByTestId("success-alert").should( + "contain", + "11 assets from Google Tag Manager have been added to the system inventory.", + ); }); }); }); diff --git a/clients/admin-ui/cypress/e2e/data-catalog.cy.ts b/clients/admin-ui/cypress/e2e/data-catalog.cy.ts new file mode 100644 index 0000000000..2eda2079b4 --- /dev/null +++ b/clients/admin-ui/cypress/e2e/data-catalog.cy.ts @@ -0,0 +1,133 @@ +import { + stubDataCatalog, + stubPlus, + stubStagedResourceActions, + stubSystemCrud, + stubTaxonomyEntities, +} from "cypress/support/stubs"; + +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; + +describe("data catalog", () => { + beforeEach(() => { + cy.login(); + stubPlus(true); + stubDataCatalog(); + stubTaxonomyEntities(); + stubSystemCrud(); + }); + + describe("systems table", () => { + beforeEach(() => { + cy.visit(DATA_CATALOG_ROUTE); + cy.wait("@getCatalogSystems"); + }); + + it("should display systems table", () => { + cy.getByTestId("row-bigquery_system-col-name").should( + "contain", + "BigQuery System", + ); + }); + + it("should be able to navigate to system details via the overflow menu", () => { + cy.getByTestId("row-bigquery_system").within(() => { + cy.getByTestId("system-actions-menu").click(); + cy.getByTestId("view-system-details").click({ force: true }); + cy.url().should("include", "/systems/configure/bigquery_system"); + }); + }); + + it("should be able to add a data use", () => { + cy.getByTestId("row-bigquery_system-col-data-uses").within(() => { + cy.getByTestId("taxonomy-add-btn").click(); + cy.get(".select-wrapper").should("be.visible"); + }); + }); + + it("should navigate to database view when clicking a system with projects", () => { + cy.getByTestId("row-bigquery_system-col-name").click(); + cy.wait("@getAvailableDatabases"); + cy.url().should("include", "/bigquery_system/projects"); + }); + + it("should navigate to dataset view when clicking a system without projects", () => { + cy.intercept("POST", "/api/v1/plus/discovery-monitor/databases*", { + fixture: "empty-pagination", + }).as("getEmptyAvailableDatabases"); + cy.getByTestId("row-dynamo_system-col-name").click(); + cy.wait("@getEmptyAvailableDatabases"); + cy.url().should("not.include", "/projects"); + }); + }); + + describe("projects table", () => { + beforeEach(() => { + cy.visit(`${DATA_CATALOG_ROUTE}/bigquery_system/projects`); + cy.wait("@getCatalogProjects"); + }); + + it("should show projects with appropriate statuses", () => { + cy.getByTestId( + "row-bigquery_monitor.prj-bigquery-111111-col-status", + ).should("contain", "Attention required"); + cy.getByTestId( + "row-bigquery_monitor.prj-bigquery-222222-col-status", + ).should("contain", "Classifying"); + cy.getByTestId( + "row-bigquery_monitor.prj-bigquery-333333-col-status", + ).should("contain", "In review"); + cy.getByTestId( + "row-bigquery_monitor.prj-bigquery-444444-col-status", + ).should("contain", "Approved"); + }); + + it("should navigate to dataset view on click", () => { + cy.getByTestId( + "row-bigquery_monitor.prj-bigquery-111111-col-name", + ).click(); + cy.url().should( + "include", + "/projects/bigquery_monitor.prj-bigquery-111111", + ); + }); + }); + + describe("resource tables", () => { + beforeEach(() => { + stubStagedResourceActions(); + cy.visit( + `${DATA_CATALOG_ROUTE}/bigquery_system/monitor.project.test_dataset_1`, + ); + }); + + it("should display the table", () => { + cy.getByTestId("row-monitor.project.dataset.table_1-col-name").should( + "contain", + "table_1", + ); + }); + + it("should be able to take actions on resources", () => { + cy.getByTestId("row-monitor.project.dataset.table_1-col-actions").within( + () => { + cy.getByTestId("classify-btn").click(); + cy.wait("@confirmResource"); + }, + ); + cy.getByTestId("row-monitor.project.dataset.table_2-col-actions").within( + () => { + cy.getByTestId("resource-actions-menu").click(); + cy.getByTestId("hide-action").click({ force: true }); + cy.wait("@ignoreResource"); + }, + ); + cy.getByTestId("row-monitor.project.dataset.table_3-col-actions").within( + () => { + cy.getByTestId("approve-btn").click(); + cy.wait("@promoteResource"); + }, + ); + }); + }); +}); diff --git a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts index b6831c64dd..cb7a3dff5c 100644 --- a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts +++ b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts @@ -410,9 +410,7 @@ describe("discovery and detection", () => { cy.intercept("PATCH", "/api/v1/plus/discovery-monitor/*/results").as( "patchClassification", ); - cy.getByTestId("classification-user.device.device_id").click({ - force: true, - }); + cy.getByTestId("classification-user.contact.phone_number").click(); cy.getByTestId("taxonomy-select").antSelect("system"); cy.wait("@patchClassification"); }); @@ -424,7 +422,7 @@ describe("discovery and detection", () => { cy.getByTestId( "user-classification-user.contact.phone_number", ).should("exist"); - cy.getByTestId("add-category-btn").click(); + cy.getByTestId("taxonomy-add-btn").click(); cy.get(".select-wrapper").should("exist"); }); }); @@ -434,7 +432,7 @@ describe("discovery and detection", () => { "row-my_bigquery_monitor.prj-bigquery-418515.test_dataset_1.consent-reports-20.No_categories-col-classifications", ).within(() => { cy.getByTestId("no-classifications").should("exist"); - cy.getByTestId("add-category-btn").should("exist"); + cy.getByTestId("taxonomy-add-btn").should("exist"); }); }); @@ -443,7 +441,7 @@ describe("discovery and detection", () => { "row-my_bigquery_monitor.prj-bigquery-418515.test_dataset_1.consent-reports-20.address-col-classifications", ).within(() => { cy.getByTestId("no-classifications").should("exist"); - cy.getByTestId("add-category-btn").should("not.exist"); + cy.getByTestId("taxonomy-add-btn").should("not.exist"); }); }); }); diff --git a/clients/admin-ui/cypress/fixtures/data-catalog/catalog-projects.json b/clients/admin-ui/cypress/fixtures/data-catalog/catalog-projects.json new file mode 100644 index 0000000000..3071598c4d --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/data-catalog/catalog-projects.json @@ -0,0 +1,32 @@ +{ + "items": [ + { + "urn": "bigquery_monitor.prj-bigquery-111111", + "name": "prj-bigquery-111111", + "diff_status": "addition", + "child_diff_status": { "addition": true } + }, + { + "urn": "bigquery_monitor.prj-bigquery-222222", + "name": "prj-bigquery-222222", + "diff_status": "classifying", + "child_diff_status": { "classifying": true } + }, + { + "urn": "bigquery_monitor.prj-bigquery-333333", + "name": "prj-bigquery-333333", + "diff_status": "classification_addition", + "child_diff_status": { "classification_addition": true } + }, + { + "urn": "bigquery_monitor.prj-bigquery-444444", + "name": "prj-bigquery-444444", + "diff_status": "monitored", + "child_diff_status": {} + } + ], + "total": 2, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/fixtures/data-catalog/catalog-systems.json b/clients/admin-ui/cypress/fixtures/data-catalog/catalog-systems.json new file mode 100644 index 0000000000..aa53ad57fc --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/data-catalog/catalog-systems.json @@ -0,0 +1,33 @@ +{ + "items": [ + { + "fides_key": "bigquery_system", + "name": "BigQuery System", + "description": "A system used for storing and analyzing large datasets.", + "monitor_config_keys": ["bigquery_monitor"], + "connection_configs": { + "key": "bq_integration" + } + }, + { + "fides_key": "dynamo_system", + "name": "Dynamo System", + "description": "A system used for storing and analyzing large datasets.", + "monitor_config_keys": ["dynamo_monitor"], + "connection_configs": { + "key": "dynamo_integration" + } + }, + { + "fides_key": "system_with_dataset", + "name": "System with Dataset", + "description": "A system with a dataset.", + "monitor_config_keys": [], + "dataset_references": ["demo_dataset"] + } + ], + "total": 3, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/fixtures/data-catalog/catalog-tables.json b/clients/admin-ui/cypress/fixtures/data-catalog/catalog-tables.json new file mode 100644 index 0000000000..56ff2e1f2f --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/data-catalog/catalog-tables.json @@ -0,0 +1,28 @@ +{ + "items": [ + { + "urn": "monitor.project.dataset.table_1", + "name": "table_1", + "diff_status": "addition" + }, + { + "urn": "monitor.project.dataset.table_2", + "name": "table_2", + "diff_status": "classifying" + }, + { + "urn": "monitor.project.dataset.table_3", + "name": "table_3", + "diff_status": "classification_addition" + }, + { + "urn": "monitor.project.dataset.table_4", + "name": "table_4", + "diff_status": "monitored" + } + ], + "total": 4, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-uncategorized.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-uncategorized.json new file mode 100644 index 0000000000..2a25b31345 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-uncategorized.json @@ -0,0 +1,897 @@ +{ + "items": [ + { + "urn": "my_web_monitor_1.forms.hubspot.com.https://forms.hubspot.com/submissions-validation/v1/validate/7252764/0d22c925-3a81-4f10-bfdc-69a5d67e93bc", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "0d22c925-3a81-4f10-bfdc-69a5d67e93bc", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "forms.hubspot.com", + "parent": "https://forms.hubspot.com/submissions-validation/v1/validate/7252764/0d22c925-3a81-4f10-bfdc-69a5d67e93bc", + "parent_domain": "forms.hubspot.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://forms.hubspot.com/submissions-validation/v1/validate/7252764/0d22c925-3a81-4f10-bfdc-69a5d67e93bc", + "cookies": [ + "__cf_bm=8N07jFbWSob3GdFT6gpNXDdVwljNEmyGlkb.K9wsnPk-1738012347-1.0.1.1-x8PIfMNzHiyO5v.CHdqpv9jNiAYAIkFRHAvSMzwcISsqOXJ8IG3OTyTlZAGhZaMp3iFko47UyhA117gr2FEwgw; path=/; expires=Mon, 27-Jan-25 21:42:27 GMT; domain=.hubspot.com; HttpOnly; Secure; SameSite=None", + "_cfuvid=Ib4J0B91YJXgO7ypk.t6_eyCRXw7vu9Zi4wdUKwPSqc-1738012347935-0.0.1.1-604800000; path=/; domain=.hubspot.com; HttpOnly; Secure; SameSite=None" + ], + "mime_type": "text/plain", + "method": "OPTIONS" + }, + { + "urn": "my_web_monitor_1.kit.fontawesome.com.https://kit.fontawesome.com/3d8d968f3c.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "3d8d968f3c.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "kit.fontawesome.com", + "parent": "https://ethyca.com/_next/static/chunks/main-9b4a7f9990eac802.js", + "parent_domain": "ethyca.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://kit.fontawesome.com/3d8d968f3c.js", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/static/chunks/5881-f396d484b738f567.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "5881-f396d484b738f567.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Javascript tag", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/static/chunks/5881-f396d484b738f567.js", + "mime_type": "application/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/static/chunks/6120-12f8a8ebf82c149e.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "6120-12f8a8ebf82c149e.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Javascript tag", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/static/chunks/6120-12f8a8ebf82c149e.js", + "mime_type": "application/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "656e9653760f19f9.css", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "mime_type": "text/css", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.js.hs-analytics.net.https://js.hs-analytics.net/analytics/1738012200000/7252764.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "7252764.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "js.hs-analytics.net", + "parent": "https://js.hs-scripts.com/7252764.js", + "parent_domain": "js.hs-scripts.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://js.hs-analytics.net/analytics/1738012200000/7252764.js", + "cookies": [ + "__cf_bm=xkymz1kLkKgrSD9HTDHLNObgnoiqlrPpx4UjLeEoJQU-1738012347-1.0.1.1-D1otlrixtUpJ35LCrCG4CglOz1cZAqFQ3lODzllRJdeAe9B6oY2WoloOyp0594Vxeq6QCRwclvdvNlufFa8kTw; path=/; expires=Mon, 27-Jan-25 21:42:27 GMT; domain=.hs-analytics.net; HttpOnly; Secure; SameSite=None" + ], + "mime_type": "text/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.js.hs-scripts.com.https://js.hs-scripts.com/7252764.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "7252764.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Javascript tag", + "domain": "js.hs-scripts.com", + "parent": "https://www.googletagmanager.com/gtm.js?id=GTM-PZHT655", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://js.hs-scripts.com/7252764.js", + "cookies": [ + "__cf_bm=A6wh0cUubg7g6bU_kL6iRgF6f06gzufUXKwjCGJQSlU-1738012346-1.0.1.1-U63taS9CMWeoitMPHkCM6G8Ob_gnT7tjxpEaCmpiXHFcaMelxJYSyyl.k8qm46rGjJOE_GnNtM4ij4vf673sBw; path=/; expires=Mon, 27-Jan-25 21:42:26 GMT; domain=.hs-scripts.com; HttpOnly; Secure; SameSite=None" + ], + "mime_type": "application/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/static/chunks/8527-eb7fda74801e0898.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "8527-eb7fda74801e0898.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Javascript tag", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/static/chunks/8527-eb7fda74801e0898.js", + "mime_type": "application/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.www.redditstatic.com.https://www.redditstatic.com/ads/conversions-config/v1/pixel/config/a2_fmeafr1ika4z_telemetry", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "a2_fmeafr1ika4z_telemetry", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "www.redditstatic.com", + "parent": "https://www.redditstatic.com/ads/pixel.js", + "parent_domain": "www.redditstatic.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://www.redditstatic.com/ads/conversions-config/v1/pixel/config/a2_fmeafr1ika4z_telemetry", + "mime_type": "application/json", + "method": "GET" + }, + { + "urn": "my_web_monitor_1..linkedin.com.AnalyticsSyncHistory", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "AnalyticsSyncHistory", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Cookie", + "domain": ".linkedin.com", + "parent": null, + "parent_domain": null, + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "value": "AQKXtpFsKPc2CgAAAZSpm8uuZBsDIwqEXNUb2UCfGlhb-ncjid9J99mNMwoi32HW0vn4C-CciBLt-uViHHMr2A", + "path": "/" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/static/chunks/pages/_app-4cc35262f25b9dc0.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "_app-4cc35262f25b9dc0.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Javascript tag", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/static/chunks/pages/_app-4cc35262f25b9dc0.js", + "mime_type": "application/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.px.ads.linkedin.com.https://px.ads.linkedin.com/attribution_trigger", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "attribution_trigger", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "px.ads.linkedin.com", + "parent": "https://snap.licdn.com/li.lms-analytics/insight.min.js", + "parent_domain": "snap.licdn.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://px.ads.linkedin.com/attribution_trigger", + "cookies": [ + "ar_debug=1; Max-Age=2629746; Expires=Thu, 27 Feb 2025 07:41:33 GMT; Path=/; Secure; HTTPOnly; SameSite=None", + "bcookie=\"v=2&4a1c8b8b-5133-4a39-8c7c-cb57fbca9d68\"; domain=.linkedin.com; Path=/; Secure; Expires=Tue, 27-Jan-2026 21:12:27 GMT; SameSite=None", + "lidc=\"b=OGST07:s=O:r=O:a=O:p=O:g=3082:u=1:x=1:i=1738012347:t=1738098747:v=2:sig=AQEQEhUb7pVU4cjfJz0rOzamagfOsij6\"; Expires=Tue, 28 Jan 2025 21:12:27 GMT; domain=.linkedin.com; Path=/; SameSite=None; Secure" + ], + "mime_type": "application/json", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/assets/away-logo.svg", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "away-logo.svg", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Image", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/assets/away-logo.svg", + "mime_type": "image/svg+xml", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/logos/Away.svg", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "Away.svg", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Image", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/logos/Away.svg", + "mime_type": "image/svg+xml", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/logos/Axios.svg", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "Axios.svg", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Image", + "domain": "ethyca.com", + "parent": "https://ethyca.com/", + "parent_domain": "ethyca.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/logos/Axios.svg", + "mime_type": "image/svg+xml", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.js.hs-banner.com.https://js.hs-banner.com/v2/7252764/banner.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "banner.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "js.hs-banner.com", + "parent": "https://js.hs-scripts.com/7252764.js", + "parent_domain": "js.hs-scripts.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://js.hs-banner.com/v2/7252764/banner.js", + "cookies": [ + "__cf_bm=HsKGWrPoFAGyvst_5foXAsV.87x0M06tsKPqWflirDE-1738012347-1.0.1.1-7yojoXj3k67o4iaBPt5kXjXZVvSLrFJrMm33GRef9ZGBMKJhHGiw9lSI4KXGA3xmbj5rW.W8gfwZcIvokNdWkw; path=/; expires=Mon, 27-Jan-25 21:42:27 GMT; domain=.hs-banner.com; HttpOnly; Secure; SameSite=None" + ], + "mime_type": "text/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/fonts/basiersquare-medium-webfont.woff2", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "basiersquare-medium-webfont.woff2", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "parent_domain": "ethyca.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/fonts/basiersquare-medium-webfont.woff2", + "mime_type": "font/woff2", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/fonts/basiersquaremono-medium-webfont.woff2", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "basiersquaremono-medium-webfont.woff2", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/fonts/basiersquaremono-medium-webfont.woff2", + "mime_type": "font/woff2", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/fonts/basiersquaremono-regular-webfont.woff2", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "basiersquaremono-regular-webfont.woff2", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/fonts/basiersquaremono-regular-webfont.woff2", + "mime_type": "font/woff2", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/fonts/basiersquare-regular-webfont.woff2", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "basiersquare-regular-webfont.woff2", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/fonts/basiersquare-regular-webfont.woff2", + "mime_type": "font/woff2", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/fonts/basiersquare-semibold-webfont.woff2", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "basiersquare-semibold-webfont.woff2", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/css/656e9653760f19f9.css", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/fonts/basiersquare-semibold-webfont.woff2", + "mime_type": "font/woff2", + "method": "GET" + }, + { + "urn": "my_web_monitor_1..linkedin.com.bcookie", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "bcookie", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Cookie", + "domain": ".linkedin.com", + "parent": "https://www.linkedin.com/px/li_sync?redirect=https%3A%2F%2Fpx.ads.linkedin.com%2Fcollect%3Fv%3D2%26fmt%3Djs%26pid%3D1143386%26time%3D1738012346515%26li_adsId%3D7daf383b-9b41-4ab3-b7da-97e7974aa689%26url%3Dhttps%253A%252F%252Fethyca.com%252F%26cookiesTest%3Dtrue%26liSync%3Dtrue", + "parent_domain": "www.linkedin.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "value": "\"v=2&497973a1-5aa9-4644-8a52-bb47c61148a5\"", + "path": "/" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/static/chunks/pages/book-demo-3d059bd9bc2c2f15.js", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "book-demo-3d059bd9bc2c2f15.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Javascript tag", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/chunks/main-9b4a7f9990eac802.js", + "parent_domain": "ethyca.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/static/chunks/pages/book-demo-3d059bd9bc2c2f15.js", + "mime_type": "application/javascript", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.ethyca.com.https://ethyca.com/_next/data/wJA-Xtvyg9StCqagMiBVe/en/book-demo.json", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "book-demo.json", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Browser Request", + "domain": "ethyca.com", + "parent": "https://ethyca.com/_next/static/chunks/main-9b4a7f9990eac802.js", + "parent_domain": "ethyca.com", + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "base_url": "https://ethyca.com/_next/data/wJA-Xtvyg9StCqagMiBVe/en/book-demo.json", + "mime_type": "application/json", + "method": "GET" + }, + { + "urn": "my_web_monitor_1..www.linkedin.com.bscookie", + "data_uses": [], + "user_assigned_data_categories": [], + "system_key": null, + "name": "bscookie", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-27T21:12:34.987756Z", + "diff_status": "addition", + "resource_type": "Cookie", + "domain": ".www.linkedin.com", + "parent": null, + "parent_domain": null, + "locations": ["USA"], + "with_consent": false, + "vendor_id": null, + "system_id": null, + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "system": null, + "value": "\"v=1&20250127211228373f1d2b-ec12-43de-88c2-d91950972a0eAQGrNuPPSR-sAwKaeM4hsFszEwJhijGl\"", + "path": "/" + } + ], + "total": 105, + "page": 1, + "size": 25, + "pages": 5 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index df518408db..8de460c866 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -365,6 +365,17 @@ export const stubSystemVendors = () => { cy.intercept("GET", "/api/v1/plus/dictionary/system-vendors", { fixture: "systems/system-vendors.json", }).as("getSystemVendors"); + cy.intercept("POST", "/api/v1/plus/dictionary/system-vendors", { + body: { + systems: [ + { + id: "123", + name: "Test System", + fides_key: "test-system", + }, + ], + }, + }).as("postSystemVendors"); }; export const stubTranslationConfig = (enabled: boolean) => { @@ -524,13 +535,52 @@ export const stubActionCenter = () => { cy.intercept("GET", "/api/v1/plus/discovery-monitor/*/results*", { fixture: "detection-discovery/activity-center/system-asset-results", }).as("getSystemAssetResults"); + cy.intercept( + "GET", + "/api/v1/plus/discovery-monitor/*/results?resolved_system_id=%5Bundefined%5D*", + { + fixture: "detection-discovery/activity-center/system-asset-uncategorized", + }, + ).as("getSystemAssetsUncategorized"); cy.intercept("POST", "/api/v1/plus/discovery-monitor/mute*", { response: 200, }).as("ignoreAssets"); cy.intercept("POST", "/api/v1/plus/discovery-monitor/promote*", { response: 200, }).as("addAssets"); + cy.intercept("POST", "/api/v1/plus/discovery-monitor/*/mute*", { + response: 200, + }).as("ignoreMonitorResultSystem"); + cy.intercept( + "POST", + "/api/v1/plus/discovery-monitor/*/mute?resolved_system_id=%5Bundefined%5D", + { + response: 200, + }, + ).as("ignoreMonitorResultUncategorizedSystem"); + cy.intercept("POST", "/api/v1/plus/discovery-monitor/*/promote*", { + response: 200, + }).as("addMonitorResultSystem"); cy.intercept("PATCH", "/api/v1/plus/discovery-monitor/*/results", { response: 200, }).as("setAssetSystem"); }; + +export const stubDataCatalog = () => { + cy.intercept("GET", "/api/v1/plus/data-catalog/system*", { + fixture: "data-catalog/catalog-systems", + }).as("getCatalogSystems"); + cy.intercept("GET", "/api/v1/plus/data-catalog/project*", { + fixture: "data-catalog/catalog-projects", + }).as("getCatalogProjects"); + cy.intercept("GET", "/api/v1/plus/discovery-monitor/results?*", { + fixture: "data-catalog/catalog-tables", + }).as("getCatalogTables"); + cy.intercept("POST", "/api/v1/plus/discovery-monitor/databases*", { + items: ["test_project"], + page: 1, + size: 1, + total: 1, + pages: 1, + }).as("getAvailableDatabases"); +}; diff --git a/clients/admin-ui/src/features/common/api.slice.ts b/clients/admin-ui/src/features/common/api.slice.ts index 574314dab0..b34cb38ac2 100644 --- a/clients/admin-ui/src/features/common/api.slice.ts +++ b/clients/admin-ui/src/features/common/api.slice.ts @@ -19,6 +19,8 @@ export const baseApi = createApi({ tagTypes: [ "Allow List", "Auth", + "Catalog Systems", + "Catalog Projects", "Classify Instances Datasets", "Classify Instances Systems", "Connection Type", diff --git a/clients/admin-ui/src/features/common/dropdown/DataCategorySelect.tsx b/clients/admin-ui/src/features/common/dropdown/DataCategorySelect.tsx new file mode 100644 index 0000000000..10e76a078f --- /dev/null +++ b/clients/admin-ui/src/features/common/dropdown/DataCategorySelect.tsx @@ -0,0 +1,40 @@ +import { + TaxonomySelect, + TaxonomySelectOption, + TaxonomySelectProps, +} from "~/features/common/dropdown/TaxonomySelect"; +import useTaxonomies from "~/features/common/hooks/useTaxonomies"; + +const DataCategorySelect = ({ + selectedTaxonomies, + showDisabled = false, + ...props +}: TaxonomySelectProps) => { + const { getDataCategoryDisplayNameProps, getDataCategories } = + useTaxonomies(); + + const getActiveDataCategories = () => + getDataCategories().filter((c) => c.active); + + const dataCategories = showDisabled + ? getDataCategories() + : getActiveDataCategories(); + + const options: TaxonomySelectOption[] = dataCategories + .filter((category) => !selectedTaxonomies.includes(category.fides_key)) + .map((category) => { + const { name, primaryName } = getDataCategoryDisplayNameProps( + category.fides_key, + ); + return { + value: category.fides_key, + name, + primaryName, + description: category.description || "", + }; + }); + + return ; +}; + +export default DataCategorySelect; diff --git a/clients/admin-ui/src/features/common/dropdown/DataUseSelect.tsx b/clients/admin-ui/src/features/common/dropdown/DataUseSelect.tsx new file mode 100644 index 0000000000..d61f5aabd6 --- /dev/null +++ b/clients/admin-ui/src/features/common/dropdown/DataUseSelect.tsx @@ -0,0 +1,36 @@ +import { + TaxonomySelect, + TaxonomySelectOption, + TaxonomySelectProps, +} from "~/features/common/dropdown/TaxonomySelect"; +import useTaxonomies from "~/features/common/hooks/useTaxonomies"; + +const DataUseSelect = ({ + selectedTaxonomies, + showDisabled = false, + ...props +}: TaxonomySelectProps) => { + const { getDataUseDisplayNameProps, getDataUses } = useTaxonomies(); + + const getActiveDataUses = () => getDataUses().filter((du) => du.active); + + const dataUses = showDisabled ? getDataUses() : getActiveDataUses(); + + const options: TaxonomySelectOption[] = dataUses + .filter((dataUse) => !selectedTaxonomies.includes(dataUse.fides_key)) + .map((dataUse) => { + const { name, primaryName } = getDataUseDisplayNameProps( + dataUse.fides_key, + ); + return { + value: dataUse.fides_key, + name, + primaryName, + description: dataUse.description || "", + }; + }); + + return ; +}; + +export default DataUseSelect; diff --git a/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx b/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx index 34501790e3..f24d2e1227 100644 --- a/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx +++ b/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx @@ -1,19 +1,46 @@ import { AntSelect as Select, AntSelectProps as SelectProps } from "fidesui"; -import { useCallback, useMemo, useState } from "react"; +import { + MouseEventHandler, + ReactNode, + useCallback, + useMemo, + useState, +} from "react"; import { useGetSystemsQuery } from "~/features/system/system.slice"; +import { CustomSelectOption } from "../form/CustomSelectOption"; import { debounce } from "../utils"; const OPTIONS_LIMIT = 25; +const dropdownRender = ( + menu: ReactNode, + onAddSystem: MouseEventHandler, +) => { + return ( + <> + + Add new system + + + {menu} + + ); +}; + interface SystemSelectProps extends Omit< SelectProps, "options" | "showSearch" | "filterOption" | "onSearch" - > {} + > { + onAddSystem?: MouseEventHandler; +} -export const SystemSelect = ({ ...props }: SystemSelectProps) => { +export const SystemSelect = ({ onAddSystem, ...props }: SystemSelectProps) => { const [searchValue, setSearchValue] = useState(); const { data, isFetching } = useGetSystemsQuery({ page: 1, @@ -45,6 +72,9 @@ export const SystemSelect = ({ ...props }: SystemSelectProps) => { placeholder="Search..." aria-label="Search for a system to select" dropdownStyle={{ minWidth: "500px" }} + dropdownRender={ + onAddSystem ? (menu) => dropdownRender(menu, onAddSystem) : undefined + } data-testid="system-select" {...props} showSearch diff --git a/clients/admin-ui/src/features/common/dropdown/TaxonomySelect.tsx b/clients/admin-ui/src/features/common/dropdown/TaxonomySelect.tsx index ef9450be4c..83a643a01f 100644 --- a/clients/admin-ui/src/features/common/dropdown/TaxonomySelect.tsx +++ b/clients/admin-ui/src/features/common/dropdown/TaxonomySelect.tsx @@ -4,7 +4,6 @@ import { AntSelectProps as SelectProps, } from "fidesui"; -import useTaxonomies from "../hooks/useTaxonomies"; import styles from "./TaxonomySelect.module.scss"; export interface TaxonomySelectOption { @@ -15,7 +14,7 @@ export interface TaxonomySelectOption { className?: string; } -const TaxonomyOption = ({ data }: { data: TaxonomySelectOption }) => { +export const TaxonomyOption = ({ data }: { data: TaxonomySelectOption }) => { return ( { ); }; -interface TaxonomySelectProps - extends SelectProps { +export interface TaxonomySelectProps + extends Omit, "options"> { selectedTaxonomies: string[]; showDisabled?: boolean; } + export const TaxonomySelect = ({ - selectedTaxonomies, - showDisabled = false, + options, ...props -}: TaxonomySelectProps) => { - const { getDataCategoryDisplayNameProps, getDataCategories } = - useTaxonomies(); - - const getActiveDataCategories = () => - getDataCategories().filter((c) => c.active); - - const dataCategories = showDisabled - ? getDataCategories() - : getActiveDataCategories(); - - const options: TaxonomySelectOption[] = dataCategories - .filter((category) => !selectedTaxonomies.includes(category.fides_key)) - .map((category) => { - const { name, primaryName } = getDataCategoryDisplayNameProps( - category.fides_key, - ); - return { - value: category.fides_key, - name, - primaryName, - description: category.description || "", - className: styles.option, - }; - }); +}: SelectProps) => { + const selectOptions = options?.map((opt) => ({ + ...opt, + className: styles.option, + })); return ( + options={selectOptions} autoFocus showSearch variant="borderless" - placeholder="Select a category..." - options={options} optionRender={TaxonomyOption} dropdownStyle={{ minWidth: "500px" }} className="w-full p-0" diff --git a/clients/admin-ui/src/features/common/form/CustomSelectOption.tsx b/clients/admin-ui/src/features/common/form/CustomSelectOption.tsx new file mode 100644 index 0000000000..13c64993a1 --- /dev/null +++ b/clients/admin-ui/src/features/common/form/CustomSelectOption.tsx @@ -0,0 +1,31 @@ +import { AntButton as Button, AntButtonProps } from "fidesui"; + +interface CustomSelectOptionProps extends Omit {} + +/** + * A button that looks like an Ant Select Option for use in custom dropdowns. + * NOTE: Use with caution, as this option will not be keyboard accessible. Ideally use the Ant Select component's native menu or have a fallback for keyboard users. + * @params extend AntButton + */ +export const CustomSelectOption = ({ + children, + className, + style, + ...props +}: CustomSelectOptionProps) => { + return ( + + ); +}; diff --git a/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx b/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx index 15bc018535..7e3659a5f2 100644 --- a/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx +++ b/clients/admin-ui/src/features/common/hooks/useTaxonomies.tsx @@ -107,6 +107,10 @@ const useTaxonomies = () => { const getDataUseDisplayName = (dataUseKey: string): JSX.Element | string => getDataDisplayName(dataUseKey, getDataUseByKey, 1); + const getDataUseDisplayNameProps = ( + dataUseKey: string, + ): DataDisplayNameProps => + getDataDisplayNameProps(dataUseKey, getDataUseByKey, 1); /* Data Categories @@ -143,6 +147,7 @@ const useTaxonomies = () => { getDataUses, getDataUseByKey, getDataUseDisplayName, + getDataUseDisplayNameProps, getDataCategories, getDataCategoryByKey, getDataCategoryDisplayName, diff --git a/clients/admin-ui/src/features/common/modals/FormModal.tsx b/clients/admin-ui/src/features/common/modals/FormModal.tsx index ef3bf2fb1e..ecf13fc265 100644 --- a/clients/admin-ui/src/features/common/modals/FormModal.tsx +++ b/clients/admin-ui/src/features/common/modals/FormModal.tsx @@ -28,6 +28,7 @@ const FormModal = ({ isCentered scrollBehavior="inside" size="xl" + id="add-modal" {...props} > diff --git a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts index ed8ae94d28..13d4b0808b 100644 --- a/clients/admin-ui/src/features/common/nav/v2/nav-config.ts +++ b/clients/admin-ui/src/features/common/nav/v2/nav-config.ts @@ -66,6 +66,13 @@ export const NAV_CONFIG: NavConfigGroup[] = [ requiresFlag: "dataDiscoveryAndDetection", requiresPlus: true, }, + { + title: "Data catalog", + path: routes.DATA_CATALOG_ROUTE, + scopes: [], + requiresFlag: "dataCatalog", + requiresPlus: true, + }, ], }, { diff --git a/clients/admin-ui/src/features/common/nav/v2/routes.ts b/clients/admin-ui/src/features/common/nav/v2/routes.ts index e14dc5ef42..384df79fcd 100644 --- a/clients/admin-ui/src/features/common/nav/v2/routes.ts +++ b/clients/admin-ui/src/features/common/nav/v2/routes.ts @@ -32,6 +32,9 @@ export const DATA_DISCOVERY_ROUTE = "/data-discovery/discovery"; export const DATA_DISCOVERY_ROUTE_DETAIL = "/data-discovery/discovery/[resourceUrn]"; +// End-to-end datasets +export const DATA_CATALOG_ROUTE = "/data-catalog"; + // Privacy requests group export const DATASTORE_CONNECTION_ROUTE = "/datastore-connection"; export const PRIVACY_REQUESTS_ROUTE = "/privacy-requests"; diff --git a/clients/admin-ui/src/features/common/table/v2/FidesCell.tsx b/clients/admin-ui/src/features/common/table/v2/FidesCell.tsx index fe5e7da3d9..50c8a19c41 100644 --- a/clients/admin-ui/src/features/common/table/v2/FidesCell.tsx +++ b/clients/admin-ui/src/features/common/table/v2/FidesCell.tsx @@ -108,6 +108,7 @@ export const FidesCell = ({ height="inherit" onClick={handleCellClick} data-testid={`row-${cell.row.id}-col-${cell.column.id}`} + {...cell.column.columnDef.meta?.cellProps} > {!cell.getIsPlaceholder() || isFirstRowOfGroupedRows ? flexRender(cell.column.columnDef.cell, { diff --git a/clients/admin-ui/src/features/common/table/v2/FidesTable.tsx b/clients/admin-ui/src/features/common/table/v2/FidesTable.tsx index efbbaa5021..97f57ededf 100644 --- a/clients/admin-ui/src/features/common/table/v2/FidesTable.tsx +++ b/clients/admin-ui/src/features/common/table/v2/FidesTable.tsx @@ -22,6 +22,7 @@ import { Portal, SmallCloseIcon, Table, + TableCellProps, TableContainer, Tbody, Td, @@ -57,6 +58,7 @@ declare module "@tanstack/table-core" { showHeaderMenuWrapOption?: boolean; overflow?: "auto" | "visible" | "hidden"; disableRowClick?: boolean; + cellProps?: TableCellProps; noPadding?: boolean; onCellClick?: (row: TData) => void; } @@ -439,6 +441,7 @@ export const FidesTableV2 = ({ opacity: 1, }, }} + {...header.column.columnDef.meta?.cellProps} > void; onRemoveTaxonomy: (taxonomy: string) => void; } -const TaxonomiesPicker = ({ +const TaxonomySelectCell = ({ selectedTaxonomies, onAddTaxonomy, onRemoveTaxonomy, -}: TaxonomiesPickerProps) => { +}: TaxonomyCellProps) => { const [isAdding, setIsAdding] = useState(false); const { getDataCategoryDisplayName } = useTaxonomies(); return ( - + {selectedTaxonomies.map((category) => ( - { setIsAdding(false); @@ -90,7 +82,7 @@ const TaxonomiesPicker = ({ /> )} - + ); }; -export default TaxonomiesPicker; +export default TaxonomySelectCell; diff --git a/clients/admin-ui/src/features/data-catalog/CatalogResourceActionsCell.tsx b/clients/admin-ui/src/features/data-catalog/CatalogResourceActionsCell.tsx new file mode 100644 index 0000000000..982bf07835 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/CatalogResourceActionsCell.tsx @@ -0,0 +1,113 @@ +import { + AntButton, + AntButton as Button, + Flex, + Menu, + MenuButton, + MenuItem, + MenuList, + MoreIcon, +} from "fidesui"; + +import { useAlert } from "~/features/common/hooks"; +import { + CatalogResourceStatus, + getCatalogResourceStatus, +} from "~/features/data-catalog/utils"; +import { + useConfirmResourceMutation, + useMuteResourceMutation, + usePromoteResourceMutation, +} from "~/features/data-discovery-and-detection/discovery-detection.slice"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const CatalogResourceActionsCell = ({ + resource, +}: { + resource: StagedResourceAPIResponse; +}) => { + const { successAlert } = useAlert(); + const status = getCatalogResourceStatus(resource); + const [confirmResource, { isLoading: classifyIsLoading }] = + useConfirmResourceMutation(); + const [promoteResource, { isLoading: approveIsLoading }] = + usePromoteResourceMutation(); + const [muteResource, { isLoading: muteIsLoading }] = + useMuteResourceMutation(); + + const classifyResource = async () => { + await confirmResource({ + staged_resource_urn: resource.urn, + monitor_config_id: resource.monitor_config_id!, + unmute_children: true, + classify_monitored_resources: true, + }); + successAlert( + `Started classification on ${resource.name ?? "this resource"}`, + ); + }; + + const approveResource = async () => { + await promoteResource({ + staged_resource_urn: resource.urn, + }); + successAlert(`Approved ${resource.name ?? " resource"}`); + }; + + const hideResource = async () => { + await muteResource({ + staged_resource_urn: resource.urn, + }); + successAlert(`Hid ${resource.name ?? " resource"}`); + }; + + const anyActionIsLoading = + classifyIsLoading || approveIsLoading || muteIsLoading; + + return ( + + {status === CatalogResourceStatus.ATTENTION_REQUIRED && ( + + )} + {status === CatalogResourceStatus.IN_REVIEW && ( + + )} + + } + data-testid="resource-actions-menu" + /> + + + Hide + + + + + ); +}; + +export default CatalogResourceActionsCell; diff --git a/clients/admin-ui/src/features/data-catalog/CatalogResourceNameCell.tsx b/clients/admin-ui/src/features/data-catalog/CatalogResourceNameCell.tsx new file mode 100644 index 0000000000..ea367a6895 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/CatalogResourceNameCell.tsx @@ -0,0 +1,19 @@ +import { DefaultCell } from "~/features/common/table/v2"; +import getResourceName from "~/features/data-discovery-and-detection/utils/getResourceName"; +import resourceHasChildren from "~/features/data-discovery-and-detection/utils/resourceHasChildren"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const CatalogResourceNameCell = ({ + resource, +}: { + resource: StagedResourceAPIResponse; +}) => { + return ( + + ); +}; + +export default CatalogResourceNameCell; diff --git a/clients/admin-ui/src/features/data-catalog/CatalogStatusBadgeCell.tsx b/clients/admin-ui/src/features/data-catalog/CatalogStatusBadgeCell.tsx new file mode 100644 index 0000000000..ef1cfb4b32 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/CatalogStatusBadgeCell.tsx @@ -0,0 +1,19 @@ +import { BadgeCell } from "~/features/common/table/v2"; +import { CatalogResourceStatus } from "~/features/data-catalog/utils"; + +const STATUS_COLOR_MAP: Record = { + [CatalogResourceStatus.ATTENTION_REQUIRED]: "red", + [CatalogResourceStatus.APPROVED]: "green", + [CatalogResourceStatus.IN_REVIEW]: "yellow", + [CatalogResourceStatus.CLASSIFYING]: "blue", +}; + +const CatalogStatusBadgeCell = ({ + status, +}: { + status: CatalogResourceStatus; +}) => { + return ; +}; + +export default CatalogStatusBadgeCell; diff --git a/clients/admin-ui/src/features/data-catalog/data-catalog.slice.ts b/clients/admin-ui/src/features/data-catalog/data-catalog.slice.ts new file mode 100644 index 0000000000..27d8bd9bcd --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/data-catalog.slice.ts @@ -0,0 +1,72 @@ +import { createSlice } from "@reduxjs/toolkit"; + +import { baseApi } from "~/features/common/api.slice"; +import { + Page_StagedResourceAPIResponse_, + Page_SystemWithMonitorKeys_, +} from "~/types/api"; +import { PaginationQueryParams } from "~/types/common/PaginationQueryParams"; + +const initialState = { + page: 1, + pageSize: 50, +}; + +interface CatalogSystemQueryParams extends PaginationQueryParams { + show_hidden?: boolean; +} + +interface CatalogResourceQueryParams extends PaginationQueryParams { + monitor_config_ids?: string[]; + show_hidden?: boolean; +} + +const dataCatalogApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getCatalogSystems: build.query< + Page_SystemWithMonitorKeys_, + CatalogSystemQueryParams + >({ + query: (params) => ({ + method: "GET", + url: `/plus/data-catalog/system`, + params, + }), + providesTags: ["Catalog Systems", "System"], + }), + getCatalogProjects: build.query< + Page_StagedResourceAPIResponse_, + CatalogResourceQueryParams + >({ + query: ({ ...params }) => ({ + method: "GET", + url: `/plus/data-catalog/project`, + params, + }), + providesTags: ["Discovery Monitor Results"], + }), + getCatalogDatasets: build.query< + Page_StagedResourceAPIResponse_, + CatalogResourceQueryParams + >({ + query: ({ ...params }) => ({ + method: "GET", + url: `/plus/data-catalog/dataset`, + params, + }), + providesTags: ["Discovery Monitor Results"], + }), + }), +}); + +export const { + useGetCatalogSystemsQuery, + useGetCatalogProjectsQuery, + useGetCatalogDatasetsQuery, +} = dataCatalogApi; + +export const dataCatalogApiSlice = createSlice({ + name: "dataCatalog", + initialState, + reducers: {}, +}); diff --git a/clients/admin-ui/src/features/data-catalog/datasets/EmptyCatalogTableNotice.tsx b/clients/admin-ui/src/features/data-catalog/datasets/EmptyCatalogTableNotice.tsx new file mode 100644 index 0000000000..26c3536e2b --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/datasets/EmptyCatalogTableNotice.tsx @@ -0,0 +1,21 @@ +import { Text, VStack } from "fidesui"; + +const EmptyCatalogTableNotice = () => ( + + + No resources found + + You're up to date! + +); + +export default EmptyCatalogTableNotice; diff --git a/clients/admin-ui/src/features/data-catalog/datasets/useCatalogDatasetColumns.tsx b/clients/admin-ui/src/features/data-catalog/datasets/useCatalogDatasetColumns.tsx new file mode 100644 index 0000000000..11ecbc09cd --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/datasets/useCatalogDatasetColumns.tsx @@ -0,0 +1,51 @@ +/* eslint-disable react/no-unstable-nested-components */ + +import { ColumnDef, createColumnHelper } from "@tanstack/react-table"; +import { useMemo } from "react"; + +import { DefaultCell } from "~/features/common/table/v2"; +import { RelativeTimestampCell } from "~/features/common/table/v2/cells"; +import CatalogResourceNameCell from "~/features/data-catalog/CatalogResourceNameCell"; +import CatalogStatusBadgeCell from "~/features/data-catalog/CatalogStatusBadgeCell"; +import { getCatalogResourceStatus } from "~/features/data-catalog/utils"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const columnHelper = createColumnHelper(); + +const useCatalogDatasetColumns = () => { + const columns: ColumnDef[] = useMemo( + () => [ + columnHelper.accessor((row) => row.name, { + id: "name", + cell: (props) => ( + + ), + header: "Dataset", + }), + columnHelper.display({ + id: "status", + cell: ({ row }) => ( + + ), + header: "Status", + }), + columnHelper.accessor((row) => row.description, { + id: "description", + cell: (props) => , + header: "Description", + }), + columnHelper.accessor((row) => row.updated_at, { + id: "lastUpdated", + cell: (props) => , + header: "Updated", + }), + ], + [], + ); + + return columns; +}; + +export default useCatalogDatasetColumns; diff --git a/clients/admin-ui/src/features/data-catalog/projects/CatalogProjectsTable.tsx b/clients/admin-ui/src/features/data-catalog/projects/CatalogProjectsTable.tsx new file mode 100644 index 0000000000..a6e9ebb3cc --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/projects/CatalogProjectsTable.tsx @@ -0,0 +1,191 @@ +/* eslint-disable react/no-unstable-nested-components */ +import { + ColumnDef, + createColumnHelper, + getCoreRowModel, + getExpandedRowModel, + getGroupedRowModel, + RowSelectionState, + useReactTable, +} from "@tanstack/react-table"; +import { Text, VStack } from "fidesui"; +import { useRouter } from "next/router"; +import { useEffect, useMemo, useState } from "react"; + +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import { + DefaultCell, + FidesTableV2, + PaginationBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { RelativeTimestampCell } from "~/features/common/table/v2/cells"; +import CatalogResourceNameCell from "~/features/data-catalog/CatalogResourceNameCell"; +import CatalogStatusBadgeCell from "~/features/data-catalog/CatalogStatusBadgeCell"; +import { useGetCatalogProjectsQuery } from "~/features/data-catalog/data-catalog.slice"; +import { getCatalogResourceStatus } from "~/features/data-catalog/utils"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const EMPTY_RESPONSE = { + items: [], + total: 0, + page: 1, + size: 50, + pages: 1, +}; + +const EmptyTableNotice = () => ( + + + + No resources found + + + +); + +const columnHelper = createColumnHelper(); + +const CatalogProjectsTable = ({ + systemKey, + monitorConfigIds, +}: { + systemKey: string; + monitorConfigIds: string[]; +}) => { + const [rowSelectionState, setRowSelectionState] = useState( + {}, + ); + + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + } = useServerSidePagination(); + + const { + isFetching, + isLoading, + data: queryResult, + } = useGetCatalogProjectsQuery({ + page: pageIndex, + size: pageSize, + monitor_config_ids: monitorConfigIds, + }); + + const router = useRouter(); + + const { + items: data, + total: totalRows, + pages: totalPages, + } = useMemo(() => queryResult ?? EMPTY_RESPONSE, [queryResult]); + + useEffect(() => { + setTotalPages(totalPages); + }, [totalPages, setTotalPages]); + + const columns: ColumnDef[] = useMemo( + () => [ + columnHelper.accessor((row) => row.name, { + id: "name", + cell: (props) => ( + + ), + header: "Project", + }), + columnHelper.display({ + id: "status", + cell: ({ row }) => ( + + ), + header: "Status", + }), + columnHelper.accessor((row) => row.monitor_config_id, { + id: "monitorConfigId", + cell: (props) => , + header: "Detected by", + }), + columnHelper.accessor((row) => row.description, { + id: "description", + cell: (props) => , + header: "Description", + }), + columnHelper.accessor((row) => row.updated_at, { + id: "lastUpdated", + cell: (props) => , + header: "Updated", + meta: { + cellProps: { + borderRight: "none", + }, + }, + }), + ], + [], + ); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + getGroupedRowModel: getGroupedRowModel(), + getExpandedRowModel: getExpandedRowModel(), + manualPagination: true, + columnResizeMode: "onChange", + columns, + data, + getRowId: (row) => row.urn, + onRowSelectionChange: setRowSelectionState, + state: { + rowSelection: rowSelectionState, + }, + }); + + if (isLoading || isFetching) { + return ; + } + + return ( + <> + } + onRowClick={(row) => + router.push(`${DATA_CATALOG_ROUTE}/${systemKey}/projects/${row.urn}`) + } + /> + + + ); +}; + +export default CatalogProjectsTable; diff --git a/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx b/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx new file mode 100644 index 0000000000..bcb8b9600e --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/staged-resources/CatalogResourcesTable.tsx @@ -0,0 +1,157 @@ +/* eslint-disable react/no-unstable-nested-components */ +import { + getCoreRowModel, + getExpandedRowModel, + getGroupedRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { Box, Flex } from "fidesui"; +import { useRouter } from "next/router"; +import { useEffect, useMemo, useState } from "react"; + +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import { + FidesTableV2, + PaginationBar, + TableActionBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import EmptyCatalogTableNotice from "~/features/data-catalog/datasets/EmptyCatalogTableNotice"; +import useCatalogResourceColumns from "~/features/data-catalog/useCatalogResourceColumns"; +import { useGetMonitorResultsQuery } from "~/features/data-discovery-and-detection/discovery-detection.slice"; +import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput"; +import { StagedResourceType } from "~/features/data-discovery-and-detection/types/StagedResourceType"; +import { findResourceType } from "~/features/data-discovery-and-detection/utils/findResourceType"; +import resourceHasChildren from "~/features/data-discovery-and-detection/utils/resourceHasChildren"; +import { + DiffStatus, + StagedResourceAPIResponse, + SystemResponse, +} from "~/types/api"; + +// everything except muted +const DIFF_STATUS_FILTERS = [ + DiffStatus.ADDITION, + DiffStatus.CLASSIFYING, + DiffStatus.CLASSIFICATION_ADDITION, + DiffStatus.CLASSIFICATION_QUEUED, + DiffStatus.CLASSIFICATION_UPDATE, + DiffStatus.MONITORED, + DiffStatus.PROMOTING, + DiffStatus.REMOVAL, + DiffStatus.REMOVING, +]; + +const EMPTY_RESPONSE = { + items: [], + total: 0, + page: 1, + size: 50, + pages: 1, +}; + +const CatalogResourcesTable = ({ + resourceUrn, + system, +}: { + resourceUrn: string; + system: SystemResponse; +}) => { + const router = useRouter(); + + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + resetPageIndexToDefault(); + }, [resourceUrn, resetPageIndexToDefault]); + + const { + isFetching, + isLoading, + data: resources, + } = useGetMonitorResultsQuery({ + staged_resource_urn: resourceUrn, + page: pageIndex, + size: pageSize, + diff_status: DIFF_STATUS_FILTERS, + search: searchQuery, + }); + + const { + items: data, + total: totalRows, + pages: totalPages, + } = useMemo(() => resources ?? EMPTY_RESPONSE, [resources]); + + useEffect(() => { + setTotalPages(totalPages); + }, [totalPages, setTotalPages]); + + const type = findResourceType(data[0] ?? StagedResourceType.NONE); + + const columns = useCatalogResourceColumns(type); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + getGroupedRowModel: getGroupedRowModel(), + getExpandedRowModel: getExpandedRowModel(), + columns, + manualPagination: true, + data, + columnResizeMode: "onChange", + getRowId: (row) => row.urn, + }); + + if (isLoading || isFetching) { + return ; + } + + return ( + <> + + + + + + + + } + getRowIsClickable={(row) => resourceHasChildren(row)} + onRowClick={(row) => + router.push(`${DATA_CATALOG_ROUTE}/${system.fides_key}/${row.urn}`) + } + /> + + + ); +}; + +export default CatalogResourcesTable; diff --git a/clients/admin-ui/src/features/data-catalog/staged-resources/parseUrnToBreadcrumbs.ts b/clients/admin-ui/src/features/data-catalog/staged-resources/parseUrnToBreadcrumbs.ts new file mode 100644 index 0000000000..17974af525 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/staged-resources/parseUrnToBreadcrumbs.ts @@ -0,0 +1,23 @@ +import { NextBreadcrumbProps } from "~/features/common/nav/v2/NextBreadcrumb"; + +const parseUrnToBreadcrumbs = ( + urn: string, + urlPrefix: string, +): NextBreadcrumbProps["items"] => { + if (!urn) { + return []; + } + const urnParts = urn.split("."); + const breadcrumbItems: NextBreadcrumbProps["items"] = []; + urnParts.reduce((prev, current) => { + const next = `${prev}.${current}`; + breadcrumbItems.push({ + title: current, + href: `${urlPrefix}/${next}`, + }); + return next; + }); + return breadcrumbItems; +}; + +export default parseUrnToBreadcrumbs; diff --git a/clients/admin-ui/src/features/data-catalog/systems/CatalogSystemsTable.tsx b/clients/admin-ui/src/features/data-catalog/systems/CatalogSystemsTable.tsx new file mode 100644 index 0000000000..bfca1bd59c --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/systems/CatalogSystemsTable.tsx @@ -0,0 +1,186 @@ +/* eslint-disable react/no-unstable-nested-components */ + +import { + ColumnDef, + createColumnHelper, + getCoreRowModel, + getExpandedRowModel, + getGroupedRowModel, + RowSelectionState, + useReactTable, +} from "@tanstack/react-table"; +import { useRouter } from "next/router"; +import { useEffect, useMemo, useState } from "react"; + +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import { + DefaultCell, + DefaultHeaderCell, + FidesTableV2, + PaginationBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { getQueryParamsFromArray } from "~/features/common/utils"; +import { useGetCatalogSystemsQuery } from "~/features/data-catalog/data-catalog.slice"; +import EmptyCatalogTableNotice from "~/features/data-catalog/datasets/EmptyCatalogTableNotice"; +import EditDataUseCell from "~/features/data-catalog/systems/EditDataUseCell"; +import SystemActionsCell from "~/features/data-catalog/systems/SystemActionCell"; +import { useLazyGetAvailableDatabasesByConnectionQuery } from "~/features/data-discovery-and-detection/discovery-detection.slice"; +import { SystemWithMonitorKeys } from "~/types/api"; + +const EMPTY_RESPONSE = { + items: [], + total: 0, + page: 1, + size: 50, + pages: 1, +}; + +const columnHelper = createColumnHelper(); + +const SystemsTable = () => { + const [rowSelectionState, setRowSelectionState] = useState( + {}, + ); + + const router = useRouter(); + + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + } = useServerSidePagination(); + + const { data: queryResult, isLoading } = useGetCatalogSystemsQuery({ + page: pageIndex, + size: pageSize, + show_hidden: false, + }); + + const [getProjects] = useLazyGetAvailableDatabasesByConnectionQuery(); + + const { + items: data, + total: totalRows, + pages: totalPages, + } = useMemo(() => queryResult ?? EMPTY_RESPONSE, [queryResult]); + + useEffect(() => { + setTotalPages(totalPages); + }, [totalPages, setTotalPages]); + + const handleRowClicked = async (row: SystemWithMonitorKeys) => { + // if there are projects, go to project view; otherwise go to datasets view + const projectsResponse = await getProjects({ + connection_config_key: row.connection_configs!.key, + page: 1, + size: 1, + }); + + const hasProjects = !!projectsResponse?.data?.total; + const queryString = getQueryParamsFromArray( + row.monitor_config_keys ?? [], + "monitor_config_ids", + ); + + const url = `${DATA_CATALOG_ROUTE}/${row.fides_key}${hasProjects ? "/projects" : ""}?${queryString}`; + router.push(url); + }; + + const columns: ColumnDef[] = useMemo( + () => [ + columnHelper.accessor((row) => row.name, { + id: "name", + cell: ({ getValue, row }) => ( + + ), + header: (props) => , + }), + columnHelper.display({ + id: "data-uses", + cell: ({ row }) => , + header: (props) => , + meta: { + disableRowClick: true, + }, + minSize: 280, + }), + columnHelper.display({ + id: "actions", + cell: (props) => ( + + router.push(`/systems/configure/${props.row.original.fides_key}`) + } + /> + ), + maxSize: 20, + enableResizing: false, + meta: { + cellProps: { + borderLeft: "none", + }, + disableRowClick: true, + }, + }), + ], + [router], + ); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + getGroupedRowModel: getGroupedRowModel(), + getExpandedRowModel: getExpandedRowModel(), + getRowId: (row) => row.fides_key, + manualPagination: true, + columnResizeMode: "onChange", + columns, + data, + onRowSelectionChange: setRowSelectionState, + state: { + rowSelection: rowSelectionState, + }, + }); + + if (isLoading) { + return ; + } + + return ( + <> + } + onRowClick={handleRowClicked} + getRowIsClickable={(row) => !!row.connection_configs?.key} + /> + + + ); +}; + +export default SystemsTable; diff --git a/clients/admin-ui/src/features/data-catalog/systems/EditDataUseCell.tsx b/clients/admin-ui/src/features/data-catalog/systems/EditDataUseCell.tsx new file mode 100644 index 0000000000..17d0a69527 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/systems/EditDataUseCell.tsx @@ -0,0 +1,125 @@ +import { + AntButton as Button, + Box, + CloseIcon, + EditIcon, + useDisclosure, +} from "fidesui"; +import { useState } from "react"; + +import DataUseSelect from "~/features/common/dropdown/DataUseSelect"; +import useTaxonomies from "~/features/common/hooks/useTaxonomies"; +import EditMinimalDataUseModal from "~/features/data-catalog/systems/EditMinimalDataUseModal"; +import TaxonomyAddButton from "~/features/data-discovery-and-detection/tables/cells/TaxonomyAddButton"; +import TaxonomyCellContainer from "~/features/data-discovery-and-detection/tables/cells/TaxonomyCellContainer"; +import TaxonomyBadge from "~/features/data-discovery-and-detection/TaxonomyBadge"; +import useSystemDataUseCrud from "~/features/data-use/useSystemDataUseCrud"; +import { + PrivacyDeclaration, + PrivacyDeclarationResponse, + SystemResponse, +} from "~/types/api"; + +interface EditDataUseCellProps { + system: SystemResponse; +} + +const DeleteDataUseButton = ({ onClick }: { onClick: () => void }) => ( + + + + + + )} + + ); +}; + +export default EditMinimalDataUseModal; diff --git a/clients/admin-ui/src/features/data-catalog/systems/SystemActionCell.tsx b/clients/admin-ui/src/features/data-catalog/systems/SystemActionCell.tsx new file mode 100644 index 0000000000..77e3ddec30 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/systems/SystemActionCell.tsx @@ -0,0 +1,39 @@ +import { + AntButton, + Menu, + MenuButton, + MenuItem, + MenuList, + MoreIcon, +} from "fidesui"; + +interface SystemActionsCellProps { + onDetailClick?: () => void; +} + +const SystemActionsCell = ({ onDetailClick }: SystemActionsCellProps) => { + return ( + + } + data-testid="system-actions-menu" + /> + + {onDetailClick && ( + + View details + + )} + + + ); +}; + +export default SystemActionsCell; diff --git a/clients/admin-ui/src/features/data-catalog/useCatalogResourceColumns.tsx b/clients/admin-ui/src/features/data-catalog/useCatalogResourceColumns.tsx new file mode 100644 index 0000000000..417485e0c4 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/useCatalogResourceColumns.tsx @@ -0,0 +1,128 @@ +import { ColumnDef, createColumnHelper } from "@tanstack/react-table"; + +import { DefaultCell } from "~/features/common/table/v2"; +import { RelativeTimestampCell } from "~/features/common/table/v2/cells"; +import CatalogResourceActionsCell from "~/features/data-catalog/CatalogResourceActionsCell"; +import CatalogResourceNameCell from "~/features/data-catalog/CatalogResourceNameCell"; +import CatalogStatusBadgeCell from "~/features/data-catalog/CatalogStatusBadgeCell"; +import { getCatalogResourceStatus } from "~/features/data-catalog/utils"; +import EditCategoryCell from "~/features/data-discovery-and-detection/tables/cells/EditCategoryCell"; +import FieldDataTypeCell from "~/features/data-discovery-and-detection/tables/cells/FieldDataTypeCell"; +import { StagedResourceType } from "~/features/data-discovery-and-detection/types/StagedResourceType"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const columnHelper = createColumnHelper(); + +const useCatalogResourceColumns = (type: StagedResourceType) => { + const defaultColumns: ColumnDef[] = []; + + if (!type) { + return defaultColumns; + } + + if (type === StagedResourceType.TABLE) { + const columnDefs = [ + columnHelper.display({ + id: "name", + cell: ({ row }) => , + header: "Table", + }), + columnHelper.display({ + id: "status", + cell: ({ row }) => ( + + ), + header: "Status", + }), + columnHelper.display({ + id: "category", + cell: ({ row }) => , + header: "Data categories", + minSize: 280, + meta: { + disableRowClick: true, + }, + }), + columnHelper.accessor((row) => row.description, { + id: "description", + cell: (props) => , + header: "Description", + }), + columnHelper.accessor((row) => row.updated_at, { + id: "lastUpdated", + cell: (props) => , + header: "Updated", + }), + columnHelper.display({ + id: "actions", + cell: ({ row }) => ( + + ), + header: "Actions", + meta: { + disableRowClick: true, + }, + }), + ]; + return columnDefs; + } + + if (type === StagedResourceType.FIELD) { + const columns = [ + columnHelper.display({ + id: "name", + cell: ({ row }) => , + header: "Field", + }), + columnHelper.display({ + id: "status", + cell: ({ row }) => ( + + ), + header: "Status", + }), + columnHelper.accessor((row) => row.data_type, { + id: "dataType", + cell: (props) => , + header: "Data type", + }), + columnHelper.display({ + id: "category", + cell: ({ row }) => , + header: "Data categories", + minSize: 280, + meta: { + disableRowClick: true, + }, + }), + columnHelper.accessor((row) => row.description, { + id: "description", + cell: (props) => , + header: "Description", + }), + columnHelper.accessor((row) => row.updated_at, { + id: "lastUpdated", + cell: (props) => , + header: "Updated", + }), + columnHelper.display({ + id: "actions", + cell: ({ row }) => ( + + ), + header: "Actions", + meta: { + disableRowClick: true, + }, + }), + ]; + return columns; + } + return defaultColumns; +}; + +export default useCatalogResourceColumns; diff --git a/clients/admin-ui/src/features/data-catalog/utils.ts b/clients/admin-ui/src/features/data-catalog/utils.ts new file mode 100644 index 0000000000..f1b2b3fcf5 --- /dev/null +++ b/clients/admin-ui/src/features/data-catalog/utils.ts @@ -0,0 +1,50 @@ +import { DiffStatus, StagedResourceAPIResponse } from "~/types/api"; + +export enum CatalogResourceStatus { + ATTENTION_REQUIRED = "Attention required", + IN_REVIEW = "In review", + APPROVED = "Approved", + CLASSIFYING = "Classifying", +} + +export const getCatalogResourceStatus = ( + resource: StagedResourceAPIResponse, +) => { + const resourceSchemaChanged = + resource.diff_status === DiffStatus.ADDITION || + resource.diff_status === DiffStatus.REMOVAL; + const resourceChildrenSchemaChanged = + resource.child_diff_statuses && + (resource.child_diff_statuses[DiffStatus.ADDITION] || + resource.child_diff_statuses[DiffStatus.REMOVAL]); + + if (resourceSchemaChanged || resourceChildrenSchemaChanged) { + return CatalogResourceStatus.ATTENTION_REQUIRED; + } + + const classificationInProgress = + resource.diff_status === DiffStatus.CLASSIFICATION_QUEUED || + resource.diff_status === DiffStatus.CLASSIFYING; + const childClassificationInProgress = + resource.child_diff_statuses && + (resource.child_diff_statuses[DiffStatus.CLASSIFICATION_QUEUED] || + resource.child_diff_statuses[DiffStatus.CLASSIFYING]); + + if (classificationInProgress || childClassificationInProgress) { + return CatalogResourceStatus.CLASSIFYING; + } + + const classificationChanged = + resource.diff_status === DiffStatus.CLASSIFICATION_ADDITION || + resource.diff_status === DiffStatus.CLASSIFICATION_UPDATE; + const childClassificationChanged = + resource.child_diff_statuses && + (resource.child_diff_statuses[DiffStatus.CLASSIFICATION_ADDITION] || + resource.child_diff_statuses[DiffStatus.CLASSIFICATION_UPDATE]); + + if (classificationChanged || childClassificationChanged) { + return CatalogResourceStatus.IN_REVIEW; + } + + return CatalogResourceStatus.APPROVED; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/TaxonomyBadge.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/TaxonomyBadge.tsx new file mode 100644 index 0000000000..067ea7893a --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/TaxonomyBadge.tsx @@ -0,0 +1,40 @@ +import { Flex, FlexProps } from "fidesui"; +import React from "react"; + +interface TaxonomyBadgeProps extends FlexProps { + children: React.ReactNode; + closeButton?: React.ReactNode; +} + +const TaxonomyBadge = ({ + children, + onClick, + closeButton, + ...props +}: TaxonomyBadgeProps) => { + return ( + + + {children} + + {closeButton} + + ); +}; + +export default TaxonomyBadge; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts index f8829abc4f..7aaab3c1b9 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts @@ -56,30 +56,48 @@ const actionCenterApi = baseApi.injectEndpoints({ }), providesTags: () => ["Discovery Monitor Results"], }), - addMonitorResults: build.mutation< + addMonitorResultSystem: build.mutation< any, - { urnList?: string[]; systemId?: string } + { monitor_config_key?: string; resolved_system_id?: string } >({ query: (params) => ({ method: "POST", - url: `/plus/discovery-monitor/promote`, + url: `/plus/discovery-monitor/${params.monitor_config_key}/promote`, params: { - staged_resource_urns: params.urnList, - system_key: params.systemId, + resolved_system_id: params.resolved_system_id, }, }), invalidatesTags: ["Discovery Monitor Results"], }), - ignoreMonitorResults: build.mutation< + ignoreMonitorResultSystem: build.mutation< any, - { urnList?: string[]; systemId?: string } + { monitor_config_key?: string; resolved_system_id?: string } >({ + query: (params) => ({ + method: "POST", + url: `/plus/discovery-monitor/${params.monitor_config_key}/mute`, + params: { + resolved_system_id: params.resolved_system_id, + }, + }), + invalidatesTags: ["Discovery Monitor Results"], + }), + addMonitorResultAssets: build.mutation({ + query: (params) => ({ + method: "POST", + url: `/plus/discovery-monitor/promote`, + params: { + staged_resource_urns: params.urnList, + }, + }), + invalidatesTags: ["Discovery Monitor Results"], + }), + ignoreMonitorResultAssets: build.mutation({ query: (params) => ({ method: "POST", url: `/plus/discovery-monitor/mute`, params: { staged_resource_urns: params.urnList, - system_key: params.systemId, }, }), invalidatesTags: ["Discovery Monitor Results"], @@ -91,6 +109,8 @@ export const { useGetAggregateMonitorResultsQuery, useGetDiscoveredSystemAggregateQuery, useGetDiscoveredAssetsQuery, - useAddMonitorResultsMutation, - useIgnoreMonitorResultsMutation, + useAddMonitorResultSystemMutation, + useIgnoreMonitorResultSystemMutation, + useAddMonitorResultAssetsMutation, + useIgnoreMonitorResultAssetsMutation, } = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredAssetsColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredAssetsColumns.tsx index 68b11c2310..3f7aa975d9 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredAssetsColumns.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredAssetsColumns.tsx @@ -30,7 +30,7 @@ export const useDiscoveredAssetsColumns = () => { dataTestId="select-all-rows" /> ), - maxSize: 25, + maxSize: 40, }), columnHelper.accessor((row) => row.name, { id: "name", @@ -47,9 +47,8 @@ export const useDiscoveredAssetsColumns = () => { cell: (props) => !!props.row.original.monitor_config_id && ( ), header: "System", diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx index b0960bfcb6..af48714c39 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx @@ -2,10 +2,11 @@ import { ColumnDef, createColumnHelper } from "@tanstack/react-table"; import { DefaultCell } from "~/features/common/table/v2"; +import { DiscoveredSystemActionsCell } from "../tables/cells/DiscoveredSystemAggregateActionsCell"; import { DiscoveredSystemStatusCell } from "../tables/cells/DiscoveredSystemAggregateStatusCell"; import { MonitorSystemAggregate } from "../types"; -export const useDiscoveredSystemAggregateColumns = () => { +export const useDiscoveredSystemAggregateColumns = (monitorId: string) => { const columnHelper = createColumnHelper(); const columns: ColumnDef[] = [ @@ -60,19 +61,20 @@ export const useDiscoveredSystemAggregateColumns = () => { ), header: "Domains", }), - /* - // TODO: [HJ-343] uncomment when actions are implemented columnHelper.display({ id: "actions", cell: (props) => ( - + ), header: "Actions", meta: { width: "auto", disableRowClick: true, }, - }), */ + }), ]; return { columns }; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredAssetsTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredAssetsTable.tsx index 84710bafd0..6b72add141 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredAssetsTable.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredAssetsTable.tsx @@ -6,6 +6,7 @@ import { import { AntButton as Button, AntEmpty as Empty, + AntTooltip as Tooltip, Flex, HStack, Icons, @@ -16,11 +17,15 @@ import { MenuList, Text, } from "fidesui"; -// import { useRouter } from "next/router"; +import { useRouter } from "next/router"; import { useEffect, useState } from "react"; +import { getErrorMessage, isErrorResult } from "~/features/common/helpers"; import { useAlert } from "~/features/common/hooks"; -// import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import { + ACTION_CENTER_ROUTE, + UNCATEGORIZED_SEGMENT, +} from "~/features/common/nav/v2/routes"; import { FidesTableV2, PaginationBar, @@ -29,9 +34,10 @@ import { useServerSidePagination, } from "~/features/common/table/v2"; import { - useAddMonitorResultsMutation, + useAddMonitorResultAssetsMutation, + useAddMonitorResultSystemMutation, useGetDiscoveredAssetsQuery, - useIgnoreMonitorResultsMutation, + useIgnoreMonitorResultAssetsMutation, } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; import { SearchInput } from "../../SearchInput"; @@ -40,20 +46,29 @@ import { useDiscoveredAssetsColumns } from "../hooks/useDiscoveredAssetsColumns" interface DiscoveredAssetsTableProps { monitorId: string; systemId: string; + onSystemName?: (name: string) => void; } export const DiscoveredAssetsTable = ({ monitorId, systemId, + onSystemName, }: DiscoveredAssetsTableProps) => { - // const router = useRouter(); + const router = useRouter(); + const [systemName, setSystemName] = useState(systemId); const [rowSelection, setRowSelection] = useState({}); - const [addMonitorResultsMutation, { isLoading: isAddingResults }] = - useAddMonitorResultsMutation(); - const [ignoreMonitorResultsMutation, { isLoading: isIgnoringResults }] = - useIgnoreMonitorResultsMutation(); + const [addMonitorResultAssetsMutation, { isLoading: isAddingResults }] = + useAddMonitorResultAssetsMutation(); + const [ignoreMonitorResultAssetsMutation, { isLoading: isIgnoringResults }] = + useIgnoreMonitorResultAssetsMutation(); + const [addMonitorResultSystemMutation, { isLoading: isAddingAllResults }] = + useAddMonitorResultSystemMutation(); + + const anyBulkActionIsLoading = + isAddingResults || isIgnoringResults || isAddingAllResults; - const anyBulkActionIsLoading = isAddingResults || isIgnoringResults; + const disableAddAll = + anyBulkActionIsLoading || systemId === UNCATEGORIZED_SEGMENT; const { PAGE_SIZES, @@ -70,8 +85,7 @@ export const DiscoveredAssetsTable = ({ resetPageIndexToDefault, } = useServerSidePagination(); const [searchQuery, setSearchQuery] = useState(""); - // const [isAddingAll, setIsAddingAll] = useState(false); - const { successAlert } = useAlert(); + const { successAlert, errorAlert } = useAlert(); useEffect(() => { resetPageIndexToDefault(); @@ -87,9 +101,12 @@ export const DiscoveredAssetsTable = ({ useEffect(() => { if (data) { + const firstSystemName = data.items[0]?.system || systemId || ""; setTotalPages(data.pages || 1); + setSystemName(firstSystemName); + onSystemName?.(firstSystemName); } - }, [data, setTotalPages]); + }, [data, systemId, onSystemName, setTotalPages]); const { columns } = useDiscoveredAssetsColumns(); @@ -105,43 +122,56 @@ export const DiscoveredAssetsTable = ({ }, }); - const selectedUrns = Object.keys(rowSelection).filter((k) => rowSelection[k]); + const selectedRows = tableInstance.getSelectedRowModel().rows; + const selectedUrns = selectedRows.map((row) => row.original.urn); const handleBulkAdd = async () => { - await addMonitorResultsMutation({ + const result = await addMonitorResultAssetsMutation({ urnList: selectedUrns, }); - // TODO: Add "view" button which will bring users to the system inventory with an asset tab open (not yet developed) - successAlert( - `${selectedUrns.length} assets from ${systemId} have been added to the system inventory.`, - `Confirmed`, - ); + if (isErrorResult(result)) { + errorAlert(getErrorMessage(result.error)); + } else { + tableInstance.resetRowSelection(); + successAlert( + `${selectedUrns.length} assets from ${systemName} have been added to the system inventory.`, + `Confirmed`, + ); + } }; const handleBulkIgnore = async () => { - await ignoreMonitorResultsMutation({ + const result = await ignoreMonitorResultAssetsMutation({ urnList: selectedUrns, }); - successAlert( - `${selectedUrns.length} assets from ${systemId} have been ignored and will not be added to the system inventory.`, - `Confirmed`, - ); + if (isErrorResult(result)) { + errorAlert(getErrorMessage(result.error)); + } else { + tableInstance.resetRowSelection(); + successAlert( + `${selectedUrns.length} assets from ${systemName} have been ignored and will not appear in future scans.`, + `Confirmed`, + ); + } }; - // TODO: [HJ-343] Uncommend when system actions are implemented - /* const handleAddAll = async () => { - setIsAddingAll(true); - await addMonitorResultsMutation({ - systemId, + const handleAddAll = async () => { + const assetCount = data?.items.length || 0; + const result = await addMonitorResultSystemMutation({ + monitor_config_key: monitorId, + resolved_system_id: systemId, }); - setIsAddingAll(false); - router.push(`${ACTION_CENTER_ROUTE}/${monitorId}`); - // TODO: Add "view" button which will bring users to the system inventory with an asset tab open (not yet developed) - successAlert( - `All assets from ${systemId} have been added to the system inventory.`, - `Confirmed`, - ); - }; */ + + if (isErrorResult(result)) { + errorAlert(getErrorMessage(result.error)); + } else { + router.push(`${ACTION_CENTER_ROUTE}/${monitorId}`); + successAlert( + `${assetCount} assets from ${systemName} have been added to the system inventory.`, + `Confirmed`, + ); + } + }; if (!monitorId || !systemId) { return null; @@ -197,19 +227,26 @@ export const DiscoveredAssetsTable = ({ - {/* - // TODO: [HJ-343] Uncommend when system actions are implemented - */} + + diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx index 0ebe3299c3..2d30b12c00 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx @@ -61,7 +61,7 @@ export const DiscoveredSystemAggregateTable = ({ } }, [data, setTotalPages]); - const { columns } = useDiscoveredSystemAggregateColumns(); + const { columns } = useDiscoveredSystemAggregateColumns(monitorId); const tableInstance = useReactTable({ getCoreRowModel: getCoreRowModel(), diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredAssetActionsCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredAssetActionsCell.tsx index 3ecb18c500..948413e621 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredAssetActionsCell.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredAssetActionsCell.tsx @@ -1,12 +1,16 @@ -import { AntButton as Button, AntSpace as Space } from "fidesui"; +import { + AntButton as Button, + AntSpace as Space, + AntTooltip as Tooltip, +} from "fidesui"; -import { isErrorResult } from "~/features/common/helpers"; +import { getErrorMessage, isErrorResult } from "~/features/common/helpers"; import { useAlert } from "~/features/common/hooks"; import { StagedResourceAPIResponse } from "~/types/api"; import { - useAddMonitorResultsMutation, - useIgnoreMonitorResultsMutation, + useAddMonitorResultAssetsMutation, + useIgnoreMonitorResultAssetsMutation, } from "../../action-center.slice"; interface DiscoveredAssetActionsCellProps { @@ -16,10 +20,10 @@ interface DiscoveredAssetActionsCellProps { export const DiscoveredAssetActionsCell = ({ asset, }: DiscoveredAssetActionsCellProps) => { - const [addMonitorResultsMutation, { isLoading: isAddingResults }] = - useAddMonitorResultsMutation(); - const [ignoreMonitorResultsMutation, { isLoading: isIgnoringResults }] = - useIgnoreMonitorResultsMutation(); + const [addMonitorResultAssetsMutation, { isLoading: isAddingResults }] = + useAddMonitorResultAssetsMutation(); + const [ignoreMonitorResultAssetsMutation, { isLoading: isIgnoringResults }] = + useIgnoreMonitorResultAssetsMutation(); const { successAlert, errorAlert } = useAlert(); @@ -28,11 +32,11 @@ export const DiscoveredAssetActionsCell = ({ const { urn, name, resource_type: type } = asset; const handleAdd = async () => { - const result = await addMonitorResultsMutation({ + const result = await addMonitorResultAssetsMutation({ urnList: [urn], }); if (isErrorResult(result)) { - errorAlert("There was adding the asset to the system inventory"); + errorAlert(getErrorMessage(result.error)); } else { successAlert( `${type} "${name}" has been added to the system inventory.`, @@ -42,26 +46,39 @@ export const DiscoveredAssetActionsCell = ({ }; const handleIgnore = async () => { - await ignoreMonitorResultsMutation({ + const result = await ignoreMonitorResultAssetsMutation({ urnList: [urn], }); - successAlert( - `${type} "${name}" has been ignored and will not be added to the system inventory.`, - `Ignored`, - ); + if (isErrorResult(result)) { + errorAlert(getErrorMessage(result.error)); + } else { + successAlert( + `${type} "${name}" has been ignored and will not appear in future scans.`, + `Ignored`, + ); + } }; + // TODO [HJ-369] update disabled and tooltip logic once the categories of consent feature is implemented return ( - + + + + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/SystemCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/SystemCell.tsx index 47231e26d5..242cc8a17f 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/SystemCell.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/SystemCell.tsx @@ -1,33 +1,49 @@ -import { AntButton, AntSelectProps, EditIcon, Icons } from "fidesui"; -import { useState } from "react"; +import { AntButton, EditIcon, Icons } from "fidesui"; +import { MouseEventHandler, useCallback, useState } from "react"; import { SystemSelect } from "~/features/common/dropdown/SystemSelect"; -import { isErrorResult } from "~/features/common/helpers"; +import { getErrorMessage, isErrorResult } from "~/features/common/helpers"; import { useAlert } from "~/features/common/hooks"; import { getTableTHandTDStyles } from "~/features/common/table/v2/util"; import ClassificationCategoryBadge from "~/features/data-discovery-and-detection/ClassificationCategoryBadge"; import { useUpdateResourceCategoryMutation } from "~/features/data-discovery-and-detection/discovery-detection.slice"; +import { AddNewSystemModal } from "~/features/system/AddNewSystemModal"; +import { StagedResourceAPIResponse } from "~/types/api"; interface SystemCellProps { - urn: string; - systemName: string | undefined | null; + aggregateSystem: StagedResourceAPIResponse; monitorConfigId: string; } export const SystemCell = ({ - urn, - systemName, + aggregateSystem, monitorConfigId, }: SystemCellProps) => { + const { + resource_type: assetType, + name: assetName, + urn, + system: systemName, + } = aggregateSystem; const [isEditing, setIsEditing] = useState(false); + const [isNewSystemModalOpen, setIsNewSystemModalOpen] = useState(false); const [updateResourceCategoryMutation, { isLoading }] = useUpdateResourceCategoryMutation(); - const { successAlert, errorAlert } = useAlert(); - const handleSelectSystem: AntSelectProps["onSelect"] = async ( + const onAddSystem: MouseEventHandler = useCallback((e) => { + e.preventDefault(); + setIsNewSystemModalOpen(true); + }, []); + + const handleCloseNewSystemModal = () => { + setIsNewSystemModalOpen(false); + }; + + const handleSelectSystem = async ( fidesKey: string, - option, + newSystemName: string, + isNewSystem?: boolean, ) => { const result = await updateResourceCategoryMutation({ staged_resource_urn: urn, @@ -35,9 +51,14 @@ export const SystemCell = ({ system_key: fidesKey, }); if (isErrorResult(result)) { - errorAlert("There was a problem the system"); + errorAlert(getErrorMessage(result.error)); } else { - successAlert(`Asset has been assigned to ${option.label}.`, `Confirmed`); + successAlert( + isNewSystem + ? `${newSystemName} has been added to your system inventory and the ${assetType} "${assetName}" has been assigned to that system.` + : `${assetType} "${assetName}" has been assigned to ${newSystemName}.`, + `Confirmed`, + ); } setIsEditing(false); }; @@ -63,6 +84,7 @@ export const SystemCell = ({ aria-label="add" icon={} onClick={() => setIsEditing(true)} + data-testid="add-system-btn" /> )} @@ -73,11 +95,28 @@ export const SystemCell = ({ className="w-full" autoFocus defaultOpen - onBlur={() => setIsEditing(false)} - onSelect={handleSelectSystem} + onBlur={(e) => { + // Close the dropdown unless the user is clicking the "Add new system" button, otherwise it won't open the modal + if (e.relatedTarget?.getAttribute("id") !== "add-new-system") { + setIsEditing(false); + } + }} + onAddSystem={onAddSystem} + onSelect={(fidesKey: string, option) => + handleSelectSystem(fidesKey, option.label as string) + } loading={isLoading} /> )} + {isNewSystemModalOpen && ( + + handleSelectSystem(fidesKey, name, true) + } + /> + )} ); }; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/discovery-detection.slice.ts b/clients/admin-ui/src/features/data-discovery-and-detection/discovery-detection.slice.ts index e12c578cbc..eaf100979f 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/discovery-detection.slice.ts +++ b/clients/admin-ui/src/features/data-discovery-and-detection/discovery-detection.slice.ts @@ -34,6 +34,7 @@ interface DatabaseByMonitorQueryParams { page: number; size: number; monitor_config_id: string; + show_hidden?: boolean; } interface DatabaseByConnectionQueryParams { @@ -90,7 +91,7 @@ const discoveryDetectionApi = baseApi.injectEndpoints({ }), }, ), - getDatabasesByConnection: build.query< + getAvailableDatabasesByConnection: build.query< Page_str_, DatabaseByConnectionQueryParams >({ @@ -230,8 +231,8 @@ export const { useGetMonitorsByIntegrationQuery, usePutDiscoveryMonitorMutation, useGetDatabasesByMonitorQuery, - useGetDatabasesByConnectionQuery, - useLazyGetDatabasesByConnectionQuery, + useGetAvailableDatabasesByConnectionQuery, + useLazyGetAvailableDatabasesByConnectionQuery, useExecuteDiscoveryMonitorMutation, useDeleteDiscoveryMonitorMutation, useGetMonitorResultsQuery, diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDetectionResultColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDetectionResultColumns.tsx index e01e585562..32a451bba9 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDetectionResultColumns.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDetectionResultColumns.tsx @@ -4,8 +4,8 @@ import { DefaultCell, DefaultHeaderCell } from "~/features/common/table/v2"; import { RelativeTimestampCell } from "~/features/common/table/v2/cells"; import DetectionItemActionsCell from "~/features/data-discovery-and-detection/tables/cells/DetectionItemActionsCell"; import FieldDataTypeCell from "~/features/data-discovery-and-detection/tables/cells/FieldDataTypeCell"; -import ResultStatusBadgeCell from "~/features/data-discovery-and-detection/tables/cells/ResultStatusBadgeCell"; import ResultStatusCell from "~/features/data-discovery-and-detection/tables/cells/ResultStatusCell"; +import ResultStatusBadgeCell from "~/features/data-discovery-and-detection/tables/cells/StagedResourceStatusBadgeCell"; import { DiscoveryMonitorItem } from "~/features/data-discovery-and-detection/types/DiscoveryMonitorItem"; import { ResourceChangeType } from "~/features/data-discovery-and-detection/types/ResourceChangeType"; import { StagedResourceType } from "~/features/data-discovery-and-detection/types/StagedResourceType"; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx index c37b67bc72..624fd87109 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx @@ -6,7 +6,7 @@ import { RelativeTimestampCell, } from "~/features/common/table/v2/cells"; import FieldDataTypeCell from "~/features/data-discovery-and-detection/tables/cells/FieldDataTypeCell"; -import ResultStatusBadgeCell from "~/features/data-discovery-and-detection/tables/cells/ResultStatusBadgeCell"; +import ResultStatusBadgeCell from "~/features/data-discovery-and-detection/tables/cells/StagedResourceStatusBadgeCell"; import { DiscoveryMonitorItem } from "~/features/data-discovery-and-detection/types/DiscoveryMonitorItem"; import { ResourceChangeType } from "~/features/data-discovery-and-detection/types/ResourceChangeType"; import { StagedResourceType } from "~/features/data-discovery-and-detection/types/StagedResourceType"; @@ -14,7 +14,7 @@ import findProjectFromUrn from "~/features/data-discovery-and-detection/utils/fi import { DiffStatus } from "~/types/api"; import DiscoveryItemActionsCell from "../tables/cells/DiscoveryItemActionsCell"; -import EditCategoriesCell from "../tables/cells/EditCategoryCell"; +import EditCategoryCell from "../tables/cells/EditCategoryCell"; import ResultStatusCell from "../tables/cells/ResultStatusCell"; const useDiscoveryResultColumns = ({ @@ -101,7 +101,7 @@ const useDiscoveryResultColumns = ({ dataTestId="select-all-rows" /> ), - maxSize: 25, + maxSize: 40, }), columnHelper.accessor((row) => row.name, { id: "tables", @@ -182,7 +182,7 @@ const useDiscoveryResultColumns = ({ columnHelper.display({ id: "classifications", cell: ({ row }) => { - return ; + return ; }, meta: { overflow: "visible", disableRowClick: true }, header: "Data category", diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx index 0828ee9d2e..d661b58936 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/ActivityTable.tsx @@ -23,8 +23,8 @@ import { import { RelativeTimestampCell } from "~/features/common/table/v2/cells"; import { useGetMonitorResultsQuery } from "~/features/data-discovery-and-detection/discovery-detection.slice"; import IconLegendTooltip from "~/features/data-discovery-and-detection/IndicatorLegend"; -import ResultStatusBadgeCell from "~/features/data-discovery-and-detection/tables/cells/ResultStatusBadgeCell"; import ResultStatusCell from "~/features/data-discovery-and-detection/tables/cells/ResultStatusCell"; +import ResultStatusBadgeCell from "~/features/data-discovery-and-detection/tables/cells/StagedResourceStatusBadgeCell"; import getResourceRowName from "~/features/data-discovery-and-detection/utils/getResourceRowName"; import { DiffStatus, diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryFieldBulkActions.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryFieldBulkActions.tsx index df0f7773b0..63e9f1e626 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryFieldBulkActions.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryFieldBulkActions.tsx @@ -1,4 +1,4 @@ -import { CheckIcon, Flex, ViewOffIcon } from "fidesui"; +import { AntFlex as Flex, CheckIcon, ViewOffIcon } from "fidesui"; import ActionButton from "~/features/data-discovery-and-detection/ActionButton"; import { @@ -31,12 +31,7 @@ const DiscoveryFieldBulkActions = ({ }; return ( - +
handleIgnoreClicked([resourceUrn])} disabled={anyActionIsLoading} loading={isMuteLoading} + size="middle" />
diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryTableBulkActions.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryTableBulkActions.tsx index 9cc2590f9e..a47f149e46 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryTableBulkActions.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/DiscoveryTableBulkActions.tsx @@ -1,4 +1,4 @@ -import { CheckIcon, Flex, Text, ViewOffIcon } from "fidesui"; +import { AntFlex as Flex, CheckIcon, Text, ViewOffIcon } from "fidesui"; import ActionButton from "~/features/data-discovery-and-detection/ActionButton"; import { @@ -35,19 +35,14 @@ const DiscoveryTableBulkActions = ({ } return ( - + {`${selectedUrns.length} selected`} -
+ } @@ -65,7 +60,7 @@ const DiscoveryTableBulkActions = ({ onClick={() => handleIgnoreClicked(selectedUrns)} size="middle" /> -
+ ); }; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/EditCategoryCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/EditCategoryCell.tsx index 05310a081f..45d08ff0a4 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/EditCategoryCell.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/EditCategoryCell.tsx @@ -1,39 +1,32 @@ -import { - AntButton as Button, - AntButtonProps as ButtonProps, - Box, - CloseIcon, - EditIcon, - SmallAddIcon, - Text, - Wrap, -} from "fidesui"; +import { AntButton as Button, Box, CloseIcon, EditIcon } from "fidesui"; import { useState } from "react"; -import { TaxonomySelect } from "~/features/common/dropdown/TaxonomySelect"; +import DataCategorySelect from "~/features/common/dropdown/DataCategorySelect"; import useTaxonomies from "~/features/common/hooks/useTaxonomies"; import { SparkleIcon } from "~/features/common/Icon/SparkleIcon"; -import ClassificationCategoryBadge from "~/features/data-discovery-and-detection/ClassificationCategoryBadge"; +import TaxonomyAddButton from "~/features/data-discovery-and-detection/tables/cells/TaxonomyAddButton"; +import TaxonomyCellContainer from "~/features/data-discovery-and-detection/tables/cells/TaxonomyCellContainer"; +import TaxonomyBadge from "~/features/data-discovery-and-detection/TaxonomyBadge"; import { DiscoveryMonitorItem } from "~/features/data-discovery-and-detection/types/DiscoveryMonitorItem"; import { useUpdateResourceCategoryMutation } from "../../discovery-detection.slice"; -const AddCategoryButton = (props: ButtonProps) => ( +interface EditCategoryCellProps { + resource: DiscoveryMonitorItem; +} + +const DeleteCategoryButton = ({ onClick }: { onClick: () => void }) => ( + + + + )} + + + ); +}; diff --git a/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx b/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx index 99807fe28e..a837942cac 100644 --- a/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx +++ b/clients/admin-ui/src/features/system/system-form-declaration-tab/PrivacyDeclarationForm.tsx @@ -24,16 +24,16 @@ import { ControlledSelect } from "~/features/common/form/ControlledSelect"; import { CustomSwitch, CustomTextInput } from "~/features/common/form/inputs"; import { FormGuard } from "~/features/common/hooks/useIsAnyFormDirty"; import { selectLockedForGVL } from "~/features/system/dictionary-form/dict-suggestion.slice"; +import useLegalBasisOptions from "~/features/system/system-form-declaration-tab/useLegalBasisOptions"; +import useSpecialCategoryLegalBasisOptions from "~/features/system/system-form-declaration-tab/useSpecialCategoryLegalBasisOptions"; import SystemFormInputGroup from "~/features/system/SystemFormInputGroup"; import { DataCategory, Dataset, DataSubject, DataUse, - LegalBasisForProcessingEnum, PrivacyDeclarationResponse, ResourceTypes, - SpecialCategoryLegalBasisEnum, } from "~/types/api"; import { Cookies } from "~/types/api/models/Cookies"; @@ -121,32 +121,10 @@ export const PrivacyDeclarationFormComponents = ({ }) => { const isEditing = !!privacyDeclarationId; - const legalBasisForProcessingOptions = useMemo( - () => - ( - Object.keys(LegalBasisForProcessingEnum) as Array< - keyof typeof LegalBasisForProcessingEnum - > - ).map((key) => ({ - value: LegalBasisForProcessingEnum[key], - label: LegalBasisForProcessingEnum[key], - })), - [], - ); - - const legalBasisForSpecialCategoryOptions = useMemo( - () => - ( - Object.keys(SpecialCategoryLegalBasisEnum) as Array< - keyof typeof SpecialCategoryLegalBasisEnum - > - ).map((key) => ({ - value: SpecialCategoryLegalBasisEnum[key], - label: SpecialCategoryLegalBasisEnum[key], - })), - [], - ); + const { legalBasisOptions } = useLegalBasisOptions(); + const { specialCategoryLegalBasisOptions } = + useSpecialCategoryLegalBasisOptions(); const datasetSelectOptions = useMemo( () => allDatasets @@ -219,7 +197,7 @@ export const PrivacyDeclarationFormComponents = ({ void; } const PrivacyDeclarationFormTab = ({ @@ -48,19 +26,15 @@ const PrivacyDeclarationFormTab = ({ addButtonProps, includeCustomFields, includeCookies, - onSave, ...dataProps }: Props & DataProps) => { - const toast = useToast(); - - const [updateSystemMutationTrigger] = useUpdateSystemMutation(); - const [showForm, setShowForm] = useState(false); + const { isOpen, onClose, onOpen } = useDisclosure(); const [currentDeclaration, setCurrentDeclaration] = useState< PrivacyDeclarationResponse | undefined >(undefined); - const { isOpen: showDictionaryModal, onClose: handleCloseDictModal } = - useDisclosure(); + const { createDataUse, updateDataUse, deleteDataUse } = + useSystemDataUseCrud(system); const assignedCookies = [ ...system.privacy_declarations @@ -77,148 +51,35 @@ const PrivacyDeclarationFormTab = ({ ) : undefined; - const checkAlreadyExists = (values: PrivacyDeclarationResponse) => { - if ( - system.privacy_declarations.filter( - (d) => d.data_use === values.data_use && d.name === values.name, - ).length > 0 - ) { - toast( - errorToastParams( - "A declaration already exists with that data use in this system. Please supply a different data use.", - ), - ); - return true; - } - return false; - }; - - const handleSave = async ( - updatedDeclarations: Omit[], - isDelete?: boolean, - ) => { - // The API can return a null name, but cannot receive a null name, - // so do an additional transform here (fides#3862) - const transformedDeclarations = updatedDeclarations.map((d) => ({ - ...d, - name: d.name ?? "", - })); - const systemBodyWithDeclaration = { - ...system, - privacy_declarations: transformedDeclarations, - }; - const handleResult = ( - result: - | { data: SystemResponse } - | { error: FetchBaseQueryError | SerializedError }, - ) => { - if (isErrorResult(result)) { - const errorMsg = getErrorMessage( - result.error, - "An unexpected error occurred while updating the system. Please try again.", - ); - - toast(errorToastParams(errorMsg)); - return undefined; - } - toast.closeAll(); - toast( - successToastParams(isDelete ? "Data use deleted" : "Data use saved"), - ); - if (onSave) { - onSave(result.data); - } - return result.data.privacy_declarations; - }; - - const updateSystemResult = await updateSystemMutationTrigger( - systemBodyWithDeclaration, - ); - - return handleResult(updateSystemResult); - }; - - const handleEditDeclaration = async ( - oldDeclaration: PrivacyDeclarationResponse, - updatedDeclaration: PrivacyDeclarationResponse, - ) => { - // Do not allow editing a privacy declaration to have the same data use as one that already exists - if ( - updatedDeclaration.id !== oldDeclaration.id && - checkAlreadyExists(updatedDeclaration) - ) { - return undefined; - } - // Because the data use can change, we also need a reference to the old declaration in order to - // make sure we are replacing the proper one - const updatedDeclarations = system.privacy_declarations.map((dec) => - dec.id === oldDeclaration.id ? updatedDeclaration : dec, - ); - return handleSave(updatedDeclarations); - }; - const handleCloseForm = () => { - setShowForm(false); + onClose(); setCurrentDeclaration(undefined); }; - const handleCreateDeclaration = async ( - values: PrivacyDeclarationResponse, - ) => { - if (checkAlreadyExists(values)) { - return undefined; - } - - toast.closeAll(); - const updatedDeclarations = [...system.privacy_declarations, values]; - const res = await handleSave(updatedDeclarations); - - handleCloseForm(); - return res; - }; - const handleOpenNewForm = () => { - setShowForm(true); + onOpen(); setCurrentDeclaration(undefined); }; const handleOpenEditForm = ( declarationToEdit: PrivacyDeclarationResponse, ) => { - setShowForm(true); + onOpen(); setCurrentDeclaration(declarationToEdit); }; - const handleAcceptDictSuggestions = (suggestions: DataUseDeclaration[]) => { - const newDeclarations = suggestions.map((du) => - transformDictDataUseToDeclaration(du), - ); - - handleSave(newDeclarations); - handleCloseDictModal(); - }; - const handleSubmit = async (values: PrivacyDeclarationResponse) => { handleCloseForm(); if (currentDeclaration) { - return handleEditDeclaration(currentDeclaration, values); + return updateDataUse(currentDeclaration, values); } - return handleCreateDeclaration(values); - }; - - const handleDelete = async ( - declarationToDelete: PrivacyDeclarationResponse, - ) => { - const updatedDeclarations = system.privacy_declarations.filter( - (dec) => dec.id !== declarationToDelete.id, - ); - return handleSave(updatedDeclarations, true); + return createDataUse(values); }; // Reset the new form when the system changes (i.e. when clicking on a new datamap node) useEffect(() => { - setShowForm(false); - }, [system.fides_key]); + onClose(); + }, [onClose, system.fides_key]); return ( @@ -234,7 +95,7 @@ const PrivacyDeclarationFormTab = ({ declarations={system.privacy_declarations} handleAdd={handleOpenNewForm} handleEdit={handleOpenEditForm} - handleDelete={handleDelete} + handleDelete={deleteDataUse} allDataUses={dataProps.allDataUses} /> )} @@ -251,7 +112,7 @@ const PrivacyDeclarationFormTab = ({ ) : null} @@ -263,22 +124,6 @@ const PrivacyDeclarationFormTab = ({ {...dataProps} /> - {system.vendor_id ? ( - - 0} - allDataUses={dataProps.allDataUses} - onCancel={handleCloseDictModal} - onAccept={handleAcceptDictSuggestions} - vendorId={system.vendor_id} - /> - - ) : null} ); }; diff --git a/clients/admin-ui/src/features/system/system-form-declaration-tab/useLegalBasisOptions.ts b/clients/admin-ui/src/features/system/system-form-declaration-tab/useLegalBasisOptions.ts new file mode 100644 index 0000000000..8ee71c1565 --- /dev/null +++ b/clients/admin-ui/src/features/system/system-form-declaration-tab/useLegalBasisOptions.ts @@ -0,0 +1,22 @@ +import { useMemo } from "react"; + +import { LegalBasisForProcessingEnum } from "~/types/api"; + +const useLegalBasisOptions = () => { + const legalBasisOptions = useMemo( + () => + ( + Object.keys(LegalBasisForProcessingEnum) as Array< + keyof typeof LegalBasisForProcessingEnum + > + ).map((key) => ({ + value: LegalBasisForProcessingEnum[key], + label: LegalBasisForProcessingEnum[key], + })), + [], + ); + + return { legalBasisOptions }; +}; + +export default useLegalBasisOptions; diff --git a/clients/admin-ui/src/features/system/system-form-declaration-tab/useSpecialCategoryLegalBasisOptions.ts b/clients/admin-ui/src/features/system/system-form-declaration-tab/useSpecialCategoryLegalBasisOptions.ts new file mode 100644 index 0000000000..6f31b9dfad --- /dev/null +++ b/clients/admin-ui/src/features/system/system-form-declaration-tab/useSpecialCategoryLegalBasisOptions.ts @@ -0,0 +1,22 @@ +import { useMemo } from "react"; + +import { SpecialCategoryLegalBasisEnum } from "~/types/api"; + +const useSpecialCategoryLegalBasisOptions = () => { + const specialCategoryLegalBasisOptions = useMemo( + () => + ( + Object.keys(SpecialCategoryLegalBasisEnum) as Array< + keyof typeof SpecialCategoryLegalBasisEnum + > + ).map((key) => ({ + value: SpecialCategoryLegalBasisEnum[key], + label: SpecialCategoryLegalBasisEnum[key], + })), + [], + ); + + return { specialCategoryLegalBasisOptions }; +}; + +export default useSpecialCategoryLegalBasisOptions; diff --git a/clients/admin-ui/src/flags.json b/clients/admin-ui/src/flags.json index 1af54fd494..380dcda4ef 100644 --- a/clients/admin-ui/src/flags.json +++ b/clients/admin-ui/src/flags.json @@ -36,6 +36,12 @@ "test": true, "production": false }, + "dataCatalog": { + "description": "Data catalog view", + "development": true, + "test": true, + "production": false + }, "webMonitor": { "description": "Monitor websites for activity", "development": true, diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/[resourceUrn].tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/[resourceUrn].tsx new file mode 100644 index 0000000000..a82d38aa17 --- /dev/null +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/[resourceUrn].tsx @@ -0,0 +1,44 @@ +import { NextPage } from "next"; +import { useRouter } from "next/router"; + +import FidesSpinner from "~/features/common/FidesSpinner"; +import Layout from "~/features/common/Layout"; +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import CatalogResourcesTable from "~/features/data-catalog/staged-resources/CatalogResourcesTable"; +import parseUrnToBreadcrumbs from "~/features/data-catalog/staged-resources/parseUrnToBreadcrumbs"; +import { useGetSystemByFidesKeyQuery } from "~/features/system"; + +const CatalogResourceView: NextPage = () => { + const { query } = useRouter(); + const systemId = query.systemId as string; + const resourceUrn = query.resourceUrn as string; + const { data: system, isLoading } = useGetSystemByFidesKeyQuery(systemId); + + const resourceBreadcrumbs = + parseUrnToBreadcrumbs(resourceUrn, `${DATA_CATALOG_ROUTE}/${systemId}`) ?? + []; + + if (isLoading) { + return ; + } + + return ( + + + + + ); +}; + +export default CatalogResourceView; diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/index.tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/index.tsx new file mode 100644 index 0000000000..4f8163317c --- /dev/null +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/index.tsx @@ -0,0 +1,127 @@ +import { + getCoreRowModel, + getExpandedRowModel, + getGroupedRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { useRouter } from "next/router"; +import { useEffect, useMemo } from "react"; + +import Layout from "~/features/common/Layout"; +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + FidesTableV2, + PaginationBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetCatalogDatasetsQuery } from "~/features/data-catalog/data-catalog.slice"; +import EmptyCatalogTableNotice from "~/features/data-catalog/datasets/EmptyCatalogTableNotice"; +import useCatalogDatasetColumns from "~/features/data-catalog/datasets/useCatalogDatasetColumns"; +import { useGetSystemByFidesKeyQuery } from "~/features/system"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const EMPTY_RESPONSE = { + items: [], + total: 0, + page: 1, + size: 50, + pages: 1, +}; + +const CatalogDatasetViewNoProjects = () => { + const { query, push } = useRouter(); + const systemKey = query.systemId as string; + const monitorConfigKeys = query.monitor_config_ids as string[]; + + const { data: system, isLoading: systemIsLoading } = + useGetSystemByFidesKeyQuery(systemKey); + + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + } = useServerSidePagination(); + + const { + isFetching, + isLoading, + data: resources, + } = useGetCatalogDatasetsQuery({ + page: pageIndex, + size: pageSize, + monitor_config_ids: monitorConfigKeys, + }); + + const { + items: data, + total: totalRows, + pages: totalPages, + } = useMemo(() => resources ?? EMPTY_RESPONSE, [resources]); + + useEffect(() => { + setTotalPages(totalPages); + }, [totalPages, setTotalPages]); + + const columns = useCatalogDatasetColumns(); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + getGroupedRowModel: getGroupedRowModel(), + getExpandedRowModel: getExpandedRowModel(), + columns, + manualPagination: true, + data, + columnResizeMode: "onChange", + }); + + const showContent = !isLoading && !systemIsLoading && !isFetching; + + return ( + + + {!showContent && } + {showContent && ( + <> + } + onRowClick={(row) => + push(`${DATA_CATALOG_ROUTE}/${systemKey}/${row.urn}`) + } + /> + + + )} + + ); +}; + +export default CatalogDatasetViewNoProjects; diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectId].tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectId].tsx new file mode 100644 index 0000000000..43d75f053a --- /dev/null +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/[projectId].tsx @@ -0,0 +1,129 @@ +import { + getCoreRowModel, + getExpandedRowModel, + getGroupedRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { useRouter } from "next/router"; +import { useEffect, useMemo } from "react"; + +import Layout from "~/features/common/Layout"; +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + FidesTableV2, + PaginationBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import EmptyCatalogTableNotice from "~/features/data-catalog/datasets/EmptyCatalogTableNotice"; +import useCatalogDatasetColumns from "~/features/data-catalog/datasets/useCatalogDatasetColumns"; +import { useGetMonitorResultsQuery } from "~/features/data-discovery-and-detection/discovery-detection.slice"; +import { useGetSystemByFidesKeyQuery } from "~/features/system"; +import { StagedResourceAPIResponse } from "~/types/api"; + +const EMPTY_RESPONSE = { + items: [], + total: 0, + page: 1, + size: 50, + pages: 1, +}; + +const CatalogDatasetView = () => { + const { query, push } = useRouter(); + const systemKey = query.systemId as string; + const projectId = query.projectId as string; + + const { data: system, isLoading: systemIsLoading } = + useGetSystemByFidesKeyQuery(systemKey); + + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + } = useServerSidePagination(); + + const { + isFetching, + isLoading, + data: resources, + } = useGetMonitorResultsQuery({ + staged_resource_urn: projectId, + page: pageIndex, + size: pageSize, + }); + + const { + items: data, + total: totalRows, + pages: totalPages, + } = useMemo(() => resources ?? EMPTY_RESPONSE, [resources]); + + useEffect(() => { + setTotalPages(totalPages); + }, [totalPages, setTotalPages]); + + const columns = useCatalogDatasetColumns(); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + getGroupedRowModel: getGroupedRowModel(), + getExpandedRowModel: getExpandedRowModel(), + columns, + manualPagination: true, + data, + columnResizeMode: "onChange", + }); + + const showContent = !isLoading && !systemIsLoading && !isFetching; + + return ( + + + {!showContent && } + {showContent && ( + <> + } + onRowClick={(row) => + push(`${DATA_CATALOG_ROUTE}/${systemKey}/${row.urn}`) + } + /> + + + )} + + ); +}; + +export default CatalogDatasetView; diff --git a/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/index.tsx b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/index.tsx new file mode 100644 index 0000000000..87fc29e80b --- /dev/null +++ b/clients/admin-ui/src/pages/data-catalog/[systemId]/projects/index.tsx @@ -0,0 +1,37 @@ +import { useRouter } from "next/router"; + +import FidesSpinner from "~/features/common/FidesSpinner"; +import Layout from "~/features/common/Layout"; +import { DATA_CATALOG_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import CatalogProjectsTable from "~/features/data-catalog/projects/CatalogProjectsTable"; +import { useGetSystemByFidesKeyQuery } from "~/features/system"; + +const CatalogProjectView = () => { + const { query } = useRouter(); + const systemKey = query.systemId as string; + const monitorConfigIds = query.monitor_config_ids as string[]; + const { data: system, isLoading } = useGetSystemByFidesKeyQuery(systemKey); + + if (isLoading) { + return ; + } + + return ( + + + + + ); +}; + +export default CatalogProjectView; diff --git a/clients/admin-ui/src/pages/data-catalog/index.tsx b/clients/admin-ui/src/pages/data-catalog/index.tsx new file mode 100644 index 0000000000..cddb2ecd8f --- /dev/null +++ b/clients/admin-ui/src/pages/data-catalog/index.tsx @@ -0,0 +1,17 @@ +import Layout from "~/features/common/Layout"; +import PageHeader from "~/features/common/PageHeader"; +import SystemsTable from "~/features/data-catalog/systems/CatalogSystemsTable"; + +const DataCatalogMainPage = () => { + return ( + + + + + ); +}; + +export default DataCatalogMainPage; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/[systemId]/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/[systemId]/index.tsx index 9d9cad6778..c580c6fdcc 100644 --- a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/[systemId]/index.tsx +++ b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/[systemId]/index.tsx @@ -1,5 +1,6 @@ import { NextPage } from "next"; import { useRouter } from "next/router"; +import { useState } from "react"; import FixedLayout from "~/features/common/FixedLayout"; import { @@ -13,6 +14,9 @@ const MonitorResultAssets: NextPage = () => { const router = useRouter(); const monitorId = decodeURIComponent(router.query.monitorId as string); const systemId = decodeURIComponent(router.query.systemId as string); + const [systemName, setSystemName] = useState( + systemId === UNCATEGORIZED_SEGMENT ? "Uncategorized assets" : systemId, + ); return ( @@ -23,11 +27,17 @@ const MonitorResultAssets: NextPage = () => { { title: monitorId, href: `${ACTION_CENTER_ROUTE}/${monitorId}` }, { title: - systemId === UNCATEGORIZED_SEGMENT ? "Uncategorized" : systemId, + systemId === UNCATEGORIZED_SEGMENT + ? "Uncategorized assets" + : systemName, }, ]} /> - + ); }; diff --git a/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/[...subfieldNames]/index.tsx b/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/[...subfieldNames]/index.tsx index cced436bb4..8d9fdf6251 100644 --- a/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/[...subfieldNames]/index.tsx +++ b/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/[...subfieldNames]/index.tsx @@ -38,7 +38,7 @@ import { TableActionBar, TableSkeletonLoader, } from "~/features/common/table/v2"; -import TaxonomiesPicker from "~/features/common/TaxonomiesPicker"; +import TaxonomySelectCell from "~/features/common/table/v2/TaxonomySelectCell"; import { DATA_BREADCRUMB_ICONS } from "~/features/data-discovery-and-detection/DiscoveryMonitorBreadcrumbs"; import { useGetDatasetByKeyQuery, @@ -199,7 +199,7 @@ const FieldsDetailPage: NextPage = () => { props.row.original.fields && props.row.original.fields?.length > 0; return ( !hasSubfields && ( - handleAddDataCategory({ dataCategory, field }) diff --git a/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/index.tsx b/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/index.tsx index 27f789f004..d7edf04050 100644 --- a/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/index.tsx +++ b/clients/admin-ui/src/pages/dataset/[datasetId]/[collectionName]/index.tsx @@ -35,7 +35,7 @@ import { TableActionBar, TableSkeletonLoader, } from "~/features/common/table/v2"; -import TaxonomiesPicker from "~/features/common/TaxonomiesPicker"; +import TaxonomySelectCell from "~/features/common/table/v2/TaxonomySelectCell"; import { DATA_BREADCRUMB_ICONS } from "~/features/data-discovery-and-detection/DiscoveryMonitorBreadcrumbs"; import { useGetDatasetByKeyQuery, @@ -182,7 +182,7 @@ const FieldsDetailPage: NextPage = () => { props.row.original.fields && props.row.original.fields?.length > 0; return ( !hasSubfields && ( - handleAddDataCategory({ dataCategory, field }) diff --git a/clients/admin-ui/src/types/api/index.ts b/clients/admin-ui/src/types/api/index.ts index c773e984da..fa6de14f84 100644 --- a/clients/admin-ui/src/types/api/index.ts +++ b/clients/admin-ui/src/types/api/index.ts @@ -28,6 +28,7 @@ export type { BasicSystemResponse } from "./models/BasicSystemResponse"; export type { BigQueryConfig } from "./models/BigQueryConfig"; export type { BigQueryDocsSchema } from "./models/BigQueryDocsSchema"; export type { Body_acquire_access_token_api_v1_oauth_token_post } from "./models/Body_acquire_access_token_api_v1_oauth_token_post"; +export type { Body_export_minimal_datamap_with_format_api_v1_plus_datamap_minimal__export_format__post } from "./models/Body_export_minimal_datamap_with_format_api_v1_plus_datamap_minimal__export_format__post"; export type { Body_upload_data_api_v1_storage__request_id__post } from "./models/Body_upload_data_api_v1_storage__request_id__post"; export type { BulkCustomFieldRequest } from "./models/BulkCustomFieldRequest"; export type { BulkPostPrivacyRequests } from "./models/BulkPostPrivacyRequests"; @@ -62,6 +63,7 @@ export type { CloudConfig } from "./models/CloudConfig"; export { ClusterHealth } from "./models/ClusterHealth"; export type { CollectionAddressResponse } from "./models/CollectionAddressResponse"; export type { CollectionMeta } from "./models/CollectionMeta"; +export type { ColumnMapItem } from "./models/ColumnMapItem"; export { ColumnSort } from "./models/ColumnSort"; export { ComponentType } from "./models/ComponentType"; export type { ConditionalValue } from "./models/ConditionalValue"; @@ -107,6 +109,7 @@ export type { Database } from "./models/Database"; export type { DatabaseConfig } from "./models/DatabaseConfig"; export type { DatabaseHealthCheck } from "./models/DatabaseHealthCheck"; export type { DataCategory } from "./models/DataCategory"; +export type { DataCategoryCreateOrUpdate } from "./models/DataCategoryCreateOrUpdate"; export type { DataFlow } from "./models/DataFlow"; export type { DatahubDocsSchema } from "./models/DatahubDocsSchema"; export { DATAMAP_GROUPING } from "./models/DATAMAP_GROUPING"; @@ -118,13 +121,16 @@ export type { DatasetConfigCtlDataset } from "./models/DatasetConfigCtlDataset"; export type { DatasetConfigSchema } from "./models/DatasetConfigSchema"; export type { DatasetField } from "./models/DatasetField"; export type { DatasetMetadata } from "./models/DatasetMetadata"; +export type { DatasetReachability } from "./models/DatasetReachability"; export type { DatasetSchema } from "./models/DatasetSchema"; export type { DatasetTraversalDetails } from "./models/DatasetTraversalDetails"; export type { DataSubject } from "./models/DataSubject"; +export type { DataSubjectCreateOrUpdate } from "./models/DataSubjectCreateOrUpdate"; export type { DataSubjectRights } from "./models/DataSubjectRights"; export { DataSubjectRightsEnum } from "./models/DataSubjectRightsEnum"; export type { DataUpload } from "./models/DataUpload"; export type { DataUse } from "./models/DataUse"; +export type { DataUseCreateOrUpdate } from "./models/DataUseCreateOrUpdate"; export { DBActions } from "./models/DBActions"; export type { DenyPrivacyRequests } from "./models/DenyPrivacyRequests"; export type { DictionaryStatus } from "./models/DictionaryStatus"; @@ -182,7 +188,9 @@ export type { FidesDocsSchema } from "./models/FidesDocsSchema"; export type { fideslang__models__Policy } from "./models/fideslang__models__Policy"; export type { FidesMeta } from "./models/FidesMeta"; export type { Field } from "./models/Field"; +export type { FieldMaskingStrategyOverride } from "./models/FieldMaskingStrategyOverride"; export type { FieldsAffectedResponse } from "./models/FieldsAffectedResponse"; +export type { FilteredPrivacyRequestResults } from "./models/FilteredPrivacyRequestResults"; export type { Generate } from "./models/Generate"; export type { GenerateRequestPayload } from "./models/GenerateRequestPayload"; export type { GenerateResponse } from "./models/GenerateResponse"; @@ -259,6 +267,7 @@ export type { MongoDBDocsSchema } from "./models/MongoDBDocsSchema"; export type { MonitorClassifyParams } from "./models/MonitorClassifyParams"; export type { MonitorConfig } from "./models/MonitorConfig"; export type { MonitorExecution } from "./models/MonitorExecution"; +export type { MonitorExecutionRequestResponse } from "./models/MonitorExecutionRequestResponse"; export { MonitorExecutionStatus } from "./models/MonitorExecutionStatus"; export { MonitorFrequency } from "./models/MonitorFrequency"; export type { MSSQLDocsSchema } from "./models/MSSQLDocsSchema"; @@ -304,6 +313,7 @@ export type { Page_StorageDestinationResponse_ } from "./models/Page_StorageDest export type { Page_str_ } from "./models/Page_str_"; export type { Page_SystemHistoryResponse_ } from "./models/Page_SystemHistoryResponse_"; export type { Page_SystemSummary_ } from "./models/Page_SystemSummary_"; +export type { Page_SystemWithMonitorKeys_ } from "./models/Page_SystemWithMonitorKeys_"; export type { Page_Union_PrivacyExperienceResponse__TCFBannerExperienceMinimalResponse__ } from "./models/Page_Union_PrivacyExperienceResponse__TCFBannerExperienceMinimalResponse__"; export type { Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__ } from "./models/Page_Union_PrivacyRequestVerboseResponse__PrivacyRequestResponse__"; export type { Page_UserResponse_ } from "./models/Page_UserResponse_"; @@ -375,6 +385,7 @@ export type { RecordConsentServedRequest } from "./models/RecordConsentServedReq export type { RecordsServedResponse } from "./models/RecordsServedResponse"; export type { RedshiftDocsSchema } from "./models/RedshiftDocsSchema"; export type { Registration } from "./models/Registration"; +export { ReportExportFormat } from "./models/ReportExportFormat"; export { ReportType } from "./models/ReportType"; export { RequestOrigin } from "./models/RequestOrigin"; export type { RequestTaskCallbackRequest } from "./models/RequestTaskCallbackRequest"; @@ -437,6 +448,7 @@ export type { SystemScanResponse } from "./models/SystemScanResponse"; export type { SystemsDiff } from "./models/SystemsDiff"; export type { SystemSummary } from "./models/SystemSummary"; export { SystemType } from "./models/SystemType"; +export type { SystemWithMonitorKeys } from "./models/SystemWithMonitorKeys"; export type { Table } from "./models/Table"; export type { TCDecode } from "./models/TCDecode"; export type { TCFBannerExperienceMinimalResponse } from "./models/TCFBannerExperienceMinimalResponse"; @@ -458,9 +470,11 @@ export type { TCFVendorRelationships } from "./models/TCFVendorRelationships"; export type { TCFVendorSave } from "./models/TCFVendorSave"; export type { TCMobileData } from "./models/TCMobileData"; export type { TestMessagingStatusMessage } from "./models/TestMessagingStatusMessage"; +export type { TestPrivacyRequest } from "./models/TestPrivacyRequest"; export { TestStatus } from "./models/TestStatus"; export type { TestStatusMessage } from "./models/TestStatusMessage"; export type { TimescaleDocsSchema } from "./models/TimescaleDocsSchema"; +export type { UnlabeledIdentities } from "./models/UnlabeledIdentities"; export { UserConsentPreference } from "./models/UserConsentPreference"; export type { UserCreate } from "./models/UserCreate"; export type { UserCreateResponse } from "./models/UserCreateResponse"; diff --git a/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts b/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts index 0581376f57..7146aced8b 100644 --- a/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/BigQueryDocsSchema.ts @@ -13,7 +13,7 @@ export type BigQueryDocsSchema = { */ keyfile_creds: fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds; /** - * The default BigQuery dataset that will be used if one isn't provided in the associated Fides datasets. + * Only provide a dataset to scope discovery monitors and privacy request automation to a specific BigQuery dataset. In most cases, this can be left blank. */ dataset?: string | null; }; diff --git a/clients/admin-ui/src/types/api/models/Body_export_minimal_datamap_with_format_api_v1_plus_datamap_minimal__export_format__post.ts b/clients/admin-ui/src/types/api/models/Body_export_minimal_datamap_with_format_api_v1_plus_datamap_minimal__export_format__post.ts new file mode 100644 index 0000000000..885603a40c --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Body_export_minimal_datamap_with_format_api_v1_plus_datamap_minimal__export_format__post.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { CustomReportCreate } from "./CustomReportCreate"; + +export type Body_export_minimal_datamap_with_format_api_v1_plus_datamap_minimal__export_format__post = + { + report?: CustomReportCreate | null; + }; diff --git a/clients/admin-ui/src/types/api/models/ColumnMapItem.ts b/clients/admin-ui/src/types/api/models/ColumnMapItem.ts new file mode 100644 index 0000000000..1bc8b2658e --- /dev/null +++ b/clients/admin-ui/src/types/api/models/ColumnMapItem.ts @@ -0,0 +1,17 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * A map between column keys and custom labels. + */ +export type ColumnMapItem = { + /** + * The custom label for the column + */ + label?: string | null; + /** + * Whether the column is shown + */ + enabled?: boolean | null; +}; diff --git a/clients/admin-ui/src/types/api/models/CustomReportConfig.ts b/clients/admin-ui/src/types/api/models/CustomReportConfig.ts index f314683bd2..7b2ad8f47b 100644 --- a/clients/admin-ui/src/types/api/models/CustomReportConfig.ts +++ b/clients/admin-ui/src/types/api/models/CustomReportConfig.ts @@ -2,7 +2,7 @@ /* tslint:disable */ /* eslint-disable */ -import type { CustomReportColumn } from "~/features/common/custom-reports/types"; +import type { ColumnMapItem } from "./ColumnMapItem"; /** * The configuration for a custom report. @@ -15,5 +15,5 @@ export type CustomReportConfig = { /** * A map between column keys and custom labels */ - column_map?: Record; + column_map?: Record; }; diff --git a/clients/admin-ui/src/types/api/models/DataCategoryCreateOrUpdate.ts b/clients/admin-ui/src/types/api/models/DataCategoryCreateOrUpdate.ts new file mode 100644 index 0000000000..692e4536f0 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DataCategoryCreateOrUpdate.ts @@ -0,0 +1,29 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type DataCategoryCreateOrUpdate = { + /** + * The version of Fideslang in which this label was added. + */ + version_added?: string | null; + /** + * The version of Fideslang in which this label was deprecated. + */ + version_deprecated?: string | null; + /** + * The new name, if applicable, for this label after deprecation. + */ + replaced_by?: string | null; + /** + * Denotes whether the resource is part of the default taxonomy or not. + */ + is_default?: boolean; + name?: string | null; + description: string | null; + active?: boolean | null; + fides_key?: string | null; + tags?: Array | null; + organization_fides_key?: string | null; + parent_key?: string | null; +}; diff --git a/clients/admin-ui/src/types/api/models/DataSubjectCreateOrUpdate.ts b/clients/admin-ui/src/types/api/models/DataSubjectCreateOrUpdate.ts new file mode 100644 index 0000000000..59f8b43985 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DataSubjectCreateOrUpdate.ts @@ -0,0 +1,32 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { DataSubjectRights } from "./DataSubjectRights"; + +export type DataSubjectCreateOrUpdate = { + /** + * The version of Fideslang in which this label was added. + */ + version_added?: string | null; + /** + * The version of Fideslang in which this label was deprecated. + */ + version_deprecated?: string | null; + /** + * The new name, if applicable, for this label after deprecation. + */ + replaced_by?: string | null; + /** + * Denotes whether the resource is part of the default taxonomy or not. + */ + is_default?: boolean; + name?: string | null; + description: string | null; + active?: boolean | null; + fides_key?: string | null; + tags?: Array | null; + organization_fides_key?: string | null; + rights?: DataSubjectRights | null; + automated_decisions_or_profiling?: boolean | null; +}; diff --git a/clients/admin-ui/src/types/api/models/DataUseCreateOrUpdate.ts b/clients/admin-ui/src/types/api/models/DataUseCreateOrUpdate.ts new file mode 100644 index 0000000000..faa216afa2 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DataUseCreateOrUpdate.ts @@ -0,0 +1,29 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type DataUseCreateOrUpdate = { + /** + * The version of Fideslang in which this label was added. + */ + version_added?: string | null; + /** + * The version of Fideslang in which this label was deprecated. + */ + version_deprecated?: string | null; + /** + * The new name, if applicable, for this label after deprecation. + */ + replaced_by?: string | null; + /** + * Denotes whether the resource is part of the default taxonomy or not. + */ + is_default?: boolean; + name?: string | null; + description: string | null; + active?: boolean | null; + fides_key?: string | null; + tags?: Array | null; + organization_fides_key?: string | null; + parent_key?: string | null; +}; diff --git a/clients/admin-ui/src/types/api/models/DatamapReport.ts b/clients/admin-ui/src/types/api/models/DatamapReport.ts index b3f8cffc13..5283343ba5 100644 --- a/clients/admin-ui/src/types/api/models/DatamapReport.ts +++ b/clients/admin-ui/src/types/api/models/DatamapReport.ts @@ -33,6 +33,7 @@ export type DatamapReport = { legal_basis_for_transfers?: Array | null; legal_name?: string | null; legitimate_interest_disclosure_url?: string | null; + link_to_processor_contract?: string | null; privacy_policy?: string | null; processes_personal_data: boolean; reason_for_exemption?: string | null; @@ -41,7 +42,9 @@ export type DatamapReport = { retention_period?: string | null; shared_categories?: Array | null; special_category_legal_basis?: string | null; + system_dependencies?: string | null; system_name: string; + third_country_safeguards?: string | null; third_parties?: string | null; uses_cookies: boolean; uses_non_cookie_access: boolean; diff --git a/clients/admin-ui/src/types/api/models/DatasetReachability.ts b/clients/admin-ui/src/types/api/models/DatasetReachability.ts new file mode 100644 index 0000000000..b53ab3307a --- /dev/null +++ b/clients/admin-ui/src/types/api/models/DatasetReachability.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Response containing reachability details for a single dataset + */ +export type DatasetReachability = { + reachable: boolean; + details: string | null; +}; diff --git a/clients/admin-ui/src/types/api/models/FidesMeta.ts b/clients/admin-ui/src/types/api/models/FidesMeta.ts index a82e7eb4e7..0973c1de13 100644 --- a/clients/admin-ui/src/types/api/models/FidesMeta.ts +++ b/clients/admin-ui/src/types/api/models/FidesMeta.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { FidesDatasetReference } from "./FidesDatasetReference"; +import type { FieldMaskingStrategyOverride } from "./FieldMaskingStrategyOverride"; /** * Supplementary metadata used by the Fides application for additional features. @@ -40,4 +41,8 @@ export type FidesMeta = { * Optionally specify that a field may be used as a custom request field in DSRs. The value is the name of the field in the DSR. */ custom_request_field?: string | null; + /** + * Optionally specify a masking strategy override for this field. + */ + masking_strategy_override?: FieldMaskingStrategyOverride | null; }; diff --git a/clients/admin-ui/src/types/api/models/FieldMaskingStrategyOverride.ts b/clients/admin-ui/src/types/api/models/FieldMaskingStrategyOverride.ts new file mode 100644 index 0000000000..d47b2b2a10 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/FieldMaskingStrategyOverride.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Overrides field-level masking strategies. + */ +export type FieldMaskingStrategyOverride = { + strategy: string; + configuration?: null; +}; diff --git a/clients/admin-ui/src/types/api/models/FilteredPrivacyRequestResults.ts b/clients/admin-ui/src/types/api/models/FilteredPrivacyRequestResults.ts new file mode 100644 index 0000000000..8205302d90 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/FilteredPrivacyRequestResults.ts @@ -0,0 +1,14 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { PrivacyRequestStatus } from "./PrivacyRequestStatus"; + +/** + * Schema representing the status and results of a test privacy request + */ +export type FilteredPrivacyRequestResults = { + privacy_request_id: string; + status: PrivacyRequestStatus; + results: string; +}; diff --git a/clients/admin-ui/src/types/api/models/MaskingStrategyOverride.ts b/clients/admin-ui/src/types/api/models/MaskingStrategyOverride.ts index ed8946c5f7..13187f1a42 100644 --- a/clients/admin-ui/src/types/api/models/MaskingStrategyOverride.ts +++ b/clients/admin-ui/src/types/api/models/MaskingStrategyOverride.ts @@ -5,7 +5,7 @@ import type { MaskingStrategies } from "./MaskingStrategies"; /** - * Overrides policy-level masking strategies. + * Overrides collection-level masking strategies. */ export type MaskingStrategyOverride = { strategy: MaskingStrategies; diff --git a/clients/admin-ui/src/types/api/models/MonitorExecutionRequestResponse.ts b/clients/admin-ui/src/types/api/models/MonitorExecutionRequestResponse.ts new file mode 100644 index 0000000000..b922d2d109 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/MonitorExecutionRequestResponse.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Schema for creating a MonitorExecution. + */ +export type MonitorExecutionRequestResponse = { + monitor_execution_id: string; +}; diff --git a/clients/admin-ui/src/types/api/models/Page_SystemWithMonitorKeys_.ts b/clients/admin-ui/src/types/api/models/Page_SystemWithMonitorKeys_.ts new file mode 100644 index 0000000000..e922a65fa6 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Page_SystemWithMonitorKeys_.ts @@ -0,0 +1,13 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { SystemWithMonitorKeys } from "./SystemWithMonitorKeys"; + +export type Page_SystemWithMonitorKeys_ = { + items: Array; + total: number | null; + page: number | null; + size: number | null; + pages?: number | null; +}; diff --git a/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts b/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts index c948edaf6e..aec25f13b6 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyRequestSource.ts @@ -9,10 +9,12 @@ * - Request Manager: Request submitted from the Admin UI's Request manager page * - Consent Webhook: Request created as a side-effect of a consent webhook request (bidirectional consent) * - Fides.js: Request created as a side-effect of a privacy preference update from Fides.js + * - Dataset Test: Standalone dataset test */ export enum PrivacyRequestSource { PRIVACY_CENTER = "Privacy Center", REQUEST_MANAGER = "Request Manager", CONSENT_WEBHOOK = "Consent Webhook", FIDES_JS = "Fides.js", + DATASET_TEST = "Dataset Test", } diff --git a/clients/admin-ui/src/types/api/models/ReportExportFormat.ts b/clients/admin-ui/src/types/api/models/ReportExportFormat.ts new file mode 100644 index 0000000000..384a86683b --- /dev/null +++ b/clients/admin-ui/src/types/api/models/ReportExportFormat.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export enum ReportExportFormat { + XLSX = "xlsx", + CSV = "csv", +} diff --git a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts index 02b1b47e18..a323692ffe 100644 --- a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts +++ b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts @@ -70,6 +70,7 @@ export enum ScopeRegistryEnum { DATASET_CREATE_OR_UPDATE = "dataset:create_or_update", DATASET_DELETE = "dataset:delete", DATASET_READ = "dataset:read", + DATASET_TEST = "dataset:test", DISCOVERY_MONITOR_READ = "discovery_monitor:read", DISCOVERY_MONITOR_UPDATE = "discovery_monitor:update", ENCRYPTION_EXEC = "encryption:exec", diff --git a/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts b/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts index b9fa1433c7..1993773b8c 100644 --- a/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/SnowflakeDocsSchema.ts @@ -31,13 +31,13 @@ export type SnowflakeDocsSchema = { */ warehouse_name: string; /** - * The name of the Snowflake database you want to connect to. + * Only provide a database name to scope discovery monitors and privacy request automation to a specific database. In most cases, this can be left blank. */ - database_name: string; + database_name?: string | null; /** - * The name of the Snowflake schema within the selected database. + * Only provide a schema to scope discovery monitors and privacy request automation to a specific schema. In most cases, this can be left blank. */ - schema_name: string; + schema_name?: string | null; /** * The Snowflake role to assume for the session, if different than Username. */ diff --git a/clients/admin-ui/src/types/api/models/SystemWithMonitorKeys.ts b/clients/admin-ui/src/types/api/models/SystemWithMonitorKeys.ts new file mode 100644 index 0000000000..d94e9b1f00 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/SystemWithMonitorKeys.ts @@ -0,0 +1,193 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationResponse"; +import type { Cookies } from "./Cookies"; +import type { DataFlow } from "./DataFlow"; +import type { DataResponsibilityTitle } from "./DataResponsibilityTitle"; +import type { LegalBasisForProfilingEnum } from "./LegalBasisForProfilingEnum"; +import type { PrivacyDeclarationResponse } from "./PrivacyDeclarationResponse"; +import type { SystemMetadata } from "./SystemMetadata"; +import type { UserResponse } from "./UserResponse"; + +/** + * Extended System response with the monitor config keys attached to the system via ConnectionConfig + */ +export type SystemWithMonitorKeys = { + /** + * A unique key used to identify this resource. + */ + fides_key: string; + /** + * Defines the Organization that this resource belongs to. + */ + organization_fides_key?: string; + tags?: Array | null; + /** + * Human-Readable name for this resource. + */ + name?: string | null; + /** + * A detailed description of what this resource is. + */ + description?: string | null; + /** + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. + */ + meta?: null; + /** + * + * The SystemMetadata resource model. + * + * Object used to hold application specific metadata for a system + * + */ + fidesctl_meta?: SystemMetadata | null; + /** + * A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc. + */ + system_type: string; + /** + * The resources to which the system sends data. + */ + egress?: Array | null; + /** + * The resources from which the system receives data. + */ + ingress?: Array | null; + /** + * Extension of base pydantic model to include DB `id` field in the response + */ + privacy_declarations: Array; + /** + * An optional value to identify the owning department or group of the system within your organization + */ + administrating_department?: string | null; + /** + * The unique identifier for the vendor that's associated with this system. + */ + vendor_id?: string | null; + /** + * If specified, the unique identifier for the vendor that was previously associated with this system. + */ + previous_vendor_id?: string | null; + /** + * The deleted date of the vendor that's associated with this system. + */ + vendor_deleted_date?: string | null; + /** + * Referenced Dataset fides keys used by the system. + */ + dataset_references?: Array; + /** + * This toggle indicates whether the system stores or processes personal data. + */ + processes_personal_data?: boolean; + /** + * This toggle indicates whether the system is exempt from privacy regulation if they do process personal data. + */ + exempt_from_privacy_regulations?: boolean; + /** + * The reason that the system is exempt from privacy regulation. + */ + reason_for_exemption?: string | null; + /** + * Whether the vendor uses data to profile a consumer in a way that has a legal effect. + */ + uses_profiling?: boolean; + /** + * The legal basis (or bases) for performing profiling that has a legal effect. + */ + legal_basis_for_profiling?: Array; + /** + * Whether this system transfers data to other countries or international organizations. + */ + does_international_transfers?: boolean; + /** + * The legal basis (or bases) under which the data is transferred. + */ + legal_basis_for_transfers?: Array; + /** + * Whether this system requires data protection impact assessments. + */ + requires_data_protection_assessments?: boolean; + /** + * Location where the DPAs or DIPAs can be found. + */ + dpa_location?: string | null; + /** + * The optional status of a Data Protection Impact Assessment + */ + dpa_progress?: string | null; + /** + * A URL that points to the system's publicly accessible privacy policy. + */ + privacy_policy?: string | null; + /** + * The legal name for the business represented by the system. + */ + legal_name?: string | null; + /** + * The legal address for the business represented by the system. + */ + legal_address?: string | null; + /** + * + * The model defining the responsibility or role over + * the system that processes personal data. + * + * Used to identify whether the organization is a + * Controller, Processor, or Sub-Processor of the data + * + */ + responsibility?: Array; + /** + * The official privacy contact address or DPO. + */ + dpo?: string | null; + /** + * The party or parties that share the responsibility for processing personal data. + */ + joint_controller_info?: string | null; + /** + * The data security practices employed by this system. + */ + data_security_practices?: string | null; + /** + * The maximum storage duration, in seconds, for cookies used by this system. + */ + cookie_max_age_seconds?: number | null; + /** + * Whether this system uses cookie storage. + */ + uses_cookies?: boolean; + /** + * Whether the system's cookies are refreshed after being initially set. + */ + cookie_refresh?: boolean; + /** + * Whether the system uses non-cookie methods of storage or accessing information stored on a user's device. + */ + uses_non_cookie_access?: boolean; + /** + * A URL that points to the system's publicly accessible legitimate interest disclosure. + */ + legitimate_interest_disclosure_url?: string | null; + /** + * System-level cookies unassociated with a data use to deliver services and functionality + */ + cookies?: Array | null; + created_at: string; + /** + * + * Describes the returned schema for a ConnectionConfiguration. + * + */ + connection_configs: ConnectionConfigurationResponse | null; + /** + * System managers of the current system + */ + data_stewards: Array | null; + monitor_config_keys?: Array; +}; diff --git a/clients/admin-ui/src/types/api/models/TestPrivacyRequest.ts b/clients/admin-ui/src/types/api/models/TestPrivacyRequest.ts new file mode 100644 index 0000000000..93af45ca8d --- /dev/null +++ b/clients/admin-ui/src/types/api/models/TestPrivacyRequest.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Schema containing the data for a test privacy request + */ +export type TestPrivacyRequest = { + privacy_request_id: string; +}; diff --git a/clients/admin-ui/src/types/api/models/UnlabeledIdentities.ts b/clients/admin-ui/src/types/api/models/UnlabeledIdentities.ts new file mode 100644 index 0000000000..124c3a81a4 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/UnlabeledIdentities.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * A model for validating identity dictionaries where standard fields use Identity's validation + * but custom fields just need to be valued. + */ +export type UnlabeledIdentities = { + data: any; +}; diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index e38c168c35..fc58e76366 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -4,7 +4,7 @@ saas_config: type: hubspot description: A sample schema representing the HubSpot connector for Fides user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/hubspot - version: 0.0.7 + version: 0.0.8 connector_params: - name: domain @@ -100,30 +100,17 @@ saas_config: - name: email identity: email update: - path: /communication-preferences/v3/unsubscribe + path: /communication-preferences/v4/statuses//unsubscribe-all method: POST body: | { - "emailAddress": "", - "subscriptionId": "", - "legalBasis": "LEGITIMATE_INTEREST_CLIENT", - "legalBasisExplanation": "At users request, we opted them out" } - data_path: subscriptionStatuses - ignore_errors: [400] param_values: - name: email identity: email - - name: subscriptionId - references: - - dataset: - field: subscription_preferences.id - direction: from - postprocessors: - - strategy: filter - configuration: - field: status - value: SUBSCRIBED + query_params: + - name: channel + value: EMAIL - name: users requests: read: diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index e82c3452af..3a675f9047 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -30,7 +30,7 @@ dataset: strategy: random_string_rewrite configuration: format_preservation: - suffix: "@company.com" + suffix: "+masked@ethyca.com" - name: firstname data_categories: [user.name] fidesops_meta: diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 6df14261d8..1e96e4e59f 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -36,7 +36,7 @@ def hubspot_secrets(saas_config): } -@pytest.fixture(scope="function") +@pytest.fixture def hubspot_identity_email(): return generate_random_email() @@ -195,15 +195,41 @@ def delete_user(self, user_id: str) -> requests.Response: ) return user_response + def opt_in_subscription_preferences( + self, email: str, subscription_id: str + ) -> requests.Response: + body = { + "emailAddress": f"{email}", + "subscriptionId": f"{subscription_id}", + "statusState": "SUBSCRIBED", + "channel": "EMAIL", + "legalBasis": "LEGITIMATE_INTEREST_CLIENT", + "legalBasisExplanation": "At users request, we opted them in", + } + subscription_response: requests.Response = requests.post( + url=f"{self.base_url}/communication-preferences/v4/statuses/{email}", + headers=self.headers, + json=body, + ) + assert subscription_response.ok + return subscription_response.json() + def get_email_subscriptions(self, email: str) -> requests.Response: email_subscriptions: requests.Response = requests.get( - url=f"{self.base_url}/communication-preferences/v3/status/email/{email}", + url=f"{self.base_url}/communication-preferences/v4/statuses/{email}?channel=EMAIL", headers=self.headers, ) return email_subscriptions + def get_all_subscriptions(self) -> requests.Response: + subscriptions_responses: requests.Response = requests.get( + url=f"{self.base_url}/communication-preferences/v4/definitions", + headers=self.headers, + ) + return subscriptions_responses.json() -@pytest.fixture(scope="function") + +@pytest.fixture def hubspot_test_client( hubspot_secrets, ) -> Generator: @@ -239,7 +265,7 @@ def user_exists(user_id: str, hubspot_test_client: HubspotTestClient) -> Any: return user_body -def create_hubspot_data(test_client, email): +def create_hubspot_data(test_client: HubspotTestClient, email): # create contact contacts_response = test_client.create_contact(email=email) contacts_body = contacts_response.json() @@ -248,6 +274,7 @@ def create_hubspot_data(test_client, email): # create user users_response = test_client.create_user(email=email) users_body = users_response.json() + user_id = users_body["id"] # no need to subscribe contact, since creating a contact auto-subscribes them @@ -268,11 +295,16 @@ def create_hubspot_data(test_client, email): (user_id, test_client), error_message=error_message, ) + subscriptions = test_client.get_all_subscriptions() + for subscription in subscriptions["results"]: + test_client.opt_in_subscription_preferences( + email=email, subscription_id=subscription["id"] + ) sleep(3) return contact_id, user_id -@pytest.fixture(scope="function") +@pytest.fixture def hubspot_data( hubspot_test_client: HubspotTestClient, hubspot_identity_email: str, diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index 34411fcf54..1eaf3e18e4 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -49,10 +49,6 @@ async def test_hubspot_access_request_task( == hubspot_identity_email ) - @pytest.mark.skip(reason="Needs troubleshooting") - @pytest.mark.usefixtures( - "use_dsr_3_0" - ) # Only testing on DSR 3.0 not 2.0 - because of fixtures taking too long to settle down async def test_hubspot_erasure_request_task( self, hubspot_runner: ConnectorRunner, @@ -66,6 +62,13 @@ async def test_hubspot_erasure_request_task( contact_id, user_id = hubspot_data + email_subscription_response = hubspot_test_client.get_email_subscriptions( + email=hubspot_identity_email + ) + subscription_body = email_subscription_response.json() + for subscription_status in subscription_body["results"]: + assert subscription_status["status"] == "SUBSCRIBED" + ( _, erasure_results, @@ -78,8 +81,8 @@ async def test_hubspot_erasure_request_task( # Masking request only issued to "contacts", "subscription_preferences", and "users" endpoints assert erasure_results == { "hubspot_instance:contacts": 1, + "hubspot_instance:subscription_preferences": 1, "hubspot_instance:owners": 0, - "hubspot_instance:subscription_preferences": 2, "hubspot_instance:users": 1, } @@ -87,15 +90,15 @@ async def test_hubspot_erasure_request_task( contact_response = hubspot_test_client.get_contact(contact_id=contact_id) contact_body = contact_response.json() assert contact_body["properties"]["firstname"] == "MASKED" - assert contact_body["properties"]["email"].endswith("@company.com") + assert contact_body["properties"]["email"].endswith("+masked@ethyca.com") # verify user is unsubscribed email_subscription_response = hubspot_test_client.get_email_subscriptions( email=hubspot_identity_email ) subscription_body = email_subscription_response.json() - for subscription_status in subscription_body["subscriptionStatuses"]: - assert subscription_status["status"] == "NOT_SUBSCRIBED" + for subscription_status in subscription_body["results"]: + assert subscription_status["status"] == "UNSUBSCRIBED" # verify user is deleted error_message = f"User with user id {user_id} could not be deleted from Hubspot"