From b805c8c8d9f06578b49b4faaeab2cb1da2226f7c Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 7 Jan 2025 09:25:43 -0800 Subject: [PATCH 01/50] Updating static checks (#5637) Co-authored-by: Adam Sachs --- .github/workflows/backend_checks.yml | 37 ------------------- .github/workflows/static_checks.yml | 53 ++++++++++++++++++++++++++++ noxfiles/ci_nox.py | 18 ++++++---- 3 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/static_checks.yml diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index e1cc215070..065a8ef920 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -75,43 +75,6 @@ jobs: path: /tmp/python-${{ matrix.python_version }}.tar retention-days: 1 - ################### - ## Static Checks ## - ################### - Static-Checks: - strategy: - matrix: - session_name: - [ - '"isort(check)"', - '"black(check)"', - "mypy", - "pylint", - "xenon", - "check_install", - '"pytest(nox)"', - ] - runs-on: ubuntu-latest - continue-on-error: true - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set Up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: "pip" - - - name: Install Nox - run: pip install nox>=2022 - - - name: Install Dev Requirements - run: pip install -r dev-requirements.txt - - - name: Run Static Check - run: nox -s ${{ matrix.session_name }} - ################## ## Performance ## ################## diff --git a/.github/workflows/static_checks.yml b/.github/workflows/static_checks.yml new file mode 100644 index 0000000000..8a26cfb6ff --- /dev/null +++ b/.github/workflows/static_checks.yml @@ -0,0 +1,53 @@ +name: Backend Static Code Checks + +on: + pull_request: + push: + branches: + - "main" + - "release-**" + +env: + IMAGE: ethyca/fides:local + DEFAULT_PYTHON_VERSION: "3.10.13" + # Docker auth with read-only permissions. + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_RO_TOKEN: ${{ secrets.DOCKER_RO_TOKEN }} + +jobs: + ################### + ## Static Checks ## + ################### + Static-Checks: + strategy: + matrix: + session_name: + [ + '"isort(check)"', + '"black(check)"', + "mypy", + "pylint", + "xenon", + "check_install", + '"pytest(nox)"', + ] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" + + - name: Install Nox + run: pip install nox>=2022 + + - name: Install Dev Requirements + run: pip install -r dev-requirements.txt + + - name: Run Static Check + run: nox -s ${{ matrix.session_name }} diff --git a/noxfiles/ci_nox.py b/noxfiles/ci_nox.py index d18428ee1d..8b835376a2 100644 --- a/noxfiles/ci_nox.py +++ b/noxfiles/ci_nox.py @@ -95,13 +95,19 @@ def xenon(session: nox.Session) -> None: "src", "tests", "scripts", - "--max-absolute B", - "--max-modules B", - "--max-average A", - "--ignore 'data, docs'", - "--exclude src/fides/_version.py", + "--max-absolute=B", + "--max-modules=B", + "--max-average=A", + "--ignore=data,docs", + "--exclude=src/fides/_version.py", + ) + session.run(*command, success_codes=[0, 1]) + session.warn( + "Note: This command was malformed so it's been failing to report complexity issues." + ) + session.warn( + "Intentionally suppressing the error status code for now to slowly work through the issues." ) - session.run(*command) ################## From 2d9f1ef27026454fac915d635d9d0b41936b0916 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Wed, 8 Jan 2025 10:49:14 -0700 Subject: [PATCH 02/50] Saved report bug fixes (#5649) --- CHANGELOG.md | 4 ++ .../admin-ui/cypress/e2e/datamap-report.cy.ts | 44 +++++++++++-- .../reporting/DatamapReportFilterModal.tsx | 2 +- .../datamap/reporting/DatamapReportTable.tsx | 65 +++++++++++++++---- .../reporting/datamap-report-context.tsx | 22 ++++--- .../src/features/datamap/reporting/utils.ts | 24 +++---- 6 files changed, 119 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 284a623081..3e7d248787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Added - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) +### Fixed +- Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) +- Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) +- Fixed issue where the Data Map report filter dialog was missing an Accordion item label [#5649](https://github.com/ethyca/fides/pull/5649) ## [2.52.0](https://github.com/ethyca/fides/compare/2.51.2...2.52.0) diff --git a/clients/admin-ui/cypress/e2e/datamap-report.cy.ts b/clients/admin-ui/cypress/e2e/datamap-report.cy.ts index 7d15b5a230..40c96cbe54 100644 --- a/clients/admin-ui/cypress/e2e/datamap-report.cy.ts +++ b/clients/admin-ui/cypress/e2e/datamap-report.cy.ts @@ -323,6 +323,15 @@ describe("Data map report table", () => { it("should filter the table by making a selection", () => { cy.getByTestId("filter-multiple-systems-btn").click(); cy.getByTestId("datamap-report-filter-modal").should("be.visible"); + cy.getByTestId("filter-modal-accordion-button") + .eq(0) + .should("have.text", "Data use"); + cy.getByTestId("filter-modal-accordion-button") + .eq(1) + .should("have.text", "Data categories"); + cy.getByTestId("filter-modal-accordion-button") + .eq(2) + .should("have.text", "Data subject"); cy.getByTestId("filter-modal-accordion-button").eq(1).click(); cy.getByTestId("filter-modal-checkbox-tree-categories").should( "be.visible", @@ -389,14 +398,15 @@ describe("Data map report table", () => { cy.get("#toast-datamap-report-toast") .should("be.visible") .should("have.attr", "data-status", "success"); - cy.getByTestId("custom-reports-trigger") - .should("contain.text", "My Custom Report") - .click(); + cy.getByTestId("custom-reports-trigger").should( + "contain.text", + "My Custom Report", + ); cy.getByTestId("fidesTable").within(() => { // reordering applied to report cy.get("thead th").eq(2).should("contain.text", "Legal name"); // column visibility applied to report - cy.get("thead th").eq(4).should("not.contain.text", "Data subject"); + cy.getByTestId("column-data_subjects").should("not.exist"); }); cy.getByTestId("group-by-menu").should( "contain.text", @@ -442,10 +452,36 @@ describe("Data map report table", () => { cy.getByTestId("custom-reports-reset-button").click(); cy.getByTestId("apply-report-button").click(); cy.getByTestId("custom-reports-popover").should("not.be.visible"); + cy.getByTestId("custom-reports-trigger").should( "contain.text", "Reports", ); + cy.getByTestId("fidesTable").within(() => { + // reordering reverted + cy.get("thead th").eq(2).should("contain.text", "Data categories"); + // column visibility restored + cy.getByTestId("column-data_subjects").should("exist"); + }); + cy.getByTestId("group-by-menu").should("contain.text", "Group by system"); + cy.getByTestId("more-menu").click(); + cy.getByTestId("edit-columns-btn").click(); + cy.get("button#data_subjects").should( + "have.attr", + "aria-checked", + "true", + ); + cy.getByTestId("column-settings-close-button").click(); + cy.getByTestId("filter-multiple-systems-btn").click(); + cy.getByTestId("datamap-report-filter-modal") + .should("be.visible") + .within(() => { + cy.getByTestId("filter-modal-accordion-button").eq(0).click(); + cy.getByTestId("checkbox-Analytics").within(() => { + cy.get("[data-checked]").should("not.exist"); + }); + cy.getByTestId("standard-dialog-close-btn").click(); + }); }); it("should allow the user cancel a report selection", () => { cy.wait("@getCustomReportsMinimal"); diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx index fc67fb016f..1bded963b2 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportFilterModal.tsx @@ -148,7 +148,7 @@ export const DatamapReportFilterModal = ({ data-testid="datamap-report-filter-modal" > - + { ], ); - useEffect(() => { - if (datamapReport?.items?.length) { - const columnIDs = Object.keys(datamapReport.items[0]); - setColumnOrder(getColumnOrder(groupBy, columnIDs)); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [groupBy, datamapReport]); - const { isOpen: isColumnSettingsOpen, onOpen: onColumnSettingsOpen, @@ -306,6 +302,20 @@ export const DatamapReportTable = () => { }, }); + useEffect(() => { + if (groupBy && !!tableInstance) { + if (tableInstance.getState().columnOrder.length === 0) { + const tableColumnIds = tableInstance.getAllColumns().map((c) => c.id); + setColumnOrder(getColumnOrder(groupBy, tableColumnIds)); + } else { + setColumnOrder( + getColumnOrder(groupBy, tableInstance.getState().columnOrder), + ); + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [groupBy, tableInstance]); + useEffect(() => { // changing the groupBy should wait until the data is loaded to update the grouping const newGrouping = getGrouping(groupBy); @@ -345,12 +355,41 @@ export const DatamapReportTable = () => { const handleSavedReport = ( savedReport: CustomReportResponse | null, - resetForm: ( + resetColumnNameForm: ( nextState?: Partial>> | undefined, ) => void, ) => { + if (!savedReport && !savedCustomReportId) { + return; + } if (!savedReport) { - setSavedCustomReportId(""); + try { + setSavedCustomReportId(""); + + /* NOTE: we can't just use tableInstance.reset() here because it will reset the table to the initial state, which is likely to include report settings that were saved in the user's local storage. Instead, we need to reset each individual setting to its default value. */ + + // reset column visibility (must happen before updating order) + setColumnVisibility(DEFAULT_COLUMN_VISIBILITY); + tableInstance.toggleAllColumnsVisible(true); + tableInstance.setColumnVisibility(DEFAULT_COLUMN_VISIBILITY); + + // reset column order (must happen prior to updating groupBy) + setColumnOrder([]); + tableInstance.setColumnOrder([]); + + // reset groupBy and filters (will automatically update the tableinstance) + setGroupBy(DATAMAP_GROUPING.SYSTEM_DATA_USE); + setSelectedFilters(DEFAULT_COLUMN_FILTERS); + + // reset column names + setColumnNameMapOverrides({}); + resetColumnNameForm({ values: {} }); + } catch (error: any) { + toast({ + status: "error", + description: "There was a problem resetting the report.", + }); + } return; } try { @@ -369,8 +408,8 @@ export const DatamapReportTable = () => { ); if (savedGroupBy) { + // No need to manually update the tableInstance here; setting the groupBy will trigger the useEffect to update the grouping. setGroupBy(savedGroupBy); - tableInstance.setGrouping(getGrouping(savedGroupBy)); } if (savedFilters) { setSelectedFilters(savedFilters); @@ -394,7 +433,7 @@ export const DatamapReportTable = () => { }, ); setColumnNameMapOverrides(columnNameMap); - resetForm({ values: columnNameMap }); + resetColumnNameForm({ values: columnNameMap }); } setSavedCustomReportId(savedReport.id); toast({ diff --git a/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx b/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx index 3051bfd243..883f331fdf 100644 --- a/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/datamap-report-context.tsx @@ -12,6 +12,17 @@ import { DATAMAP_GROUPING } from "~/types/api"; import { DatamapReportFilterSelections } from "../types"; import { COLUMN_IDS, DATAMAP_LOCAL_STORAGE_KEYS } from "./constants"; +export const DEFAULT_COLUMN_VISIBILITY = { + [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: false, + [COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES]: false, +}; + +export const DEFAULT_COLUMN_FILTERS = { + dataUses: [], + dataSubjects: [], + dataCategories: [], +}; + interface DatamapReportContextProps { savedCustomReportId: string; setSavedCustomReportId: Dispatch>; @@ -51,11 +62,7 @@ export const DatamapReportProvider = ({ const [selectedFilters, setSelectedFilters] = useLocalStorage( DATAMAP_LOCAL_STORAGE_KEYS.FILTERS, - { - dataUses: [], - dataSubjects: [], - dataCategories: [], - }, + DEFAULT_COLUMN_FILTERS, ); const [columnOrder, setColumnOrder] = useLocalStorage( @@ -65,10 +72,7 @@ export const DatamapReportProvider = ({ const [columnVisibility, setColumnVisibility] = useLocalStorage< Record - >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_VISIBILITY, { - [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: false, - [COLUMN_IDS.DATA_USE_UNDECLARED_DATA_CATEGORIES]: false, - }); + >(DATAMAP_LOCAL_STORAGE_KEYS.COLUMN_VISIBILITY, DEFAULT_COLUMN_VISIBILITY); const [columnSizing, setColumnSizing] = useLocalStorage< Record diff --git a/clients/admin-ui/src/features/datamap/reporting/utils.ts b/clients/admin-ui/src/features/datamap/reporting/utils.ts index 6c7b910134..343dce7969 100644 --- a/clients/admin-ui/src/features/datamap/reporting/utils.ts +++ b/clients/admin-ui/src/features/datamap/reporting/utils.ts @@ -12,10 +12,7 @@ export const getGrouping = (groupBy?: DATAMAP_GROUPING) => { } }; -export const getColumnOrder = ( - groupBy: DATAMAP_GROUPING, - columnIDs: string[], -) => { +export const getPrefixColumns = (groupBy: DATAMAP_GROUPING) => { let columnOrder: string[] = []; if (DATAMAP_GROUPING.SYSTEM_DATA_USE === groupBy) { columnOrder = [COLUMN_IDS.SYSTEM_NAME, COLUMN_IDS.DATA_USE]; @@ -23,6 +20,14 @@ export const getColumnOrder = ( if (DATAMAP_GROUPING.DATA_USE_SYSTEM === groupBy) { columnOrder = [COLUMN_IDS.DATA_USE, COLUMN_IDS.SYSTEM_NAME]; } + return columnOrder; +}; + +export const getColumnOrder = ( + groupBy: DATAMAP_GROUPING, + columnIDs: string[], +) => { + let columnOrder: string[] = getPrefixColumns(groupBy); columnOrder = columnOrder.concat( columnIDs.filter( (columnID) => @@ -31,14 +36,3 @@ export const getColumnOrder = ( ); return columnOrder; }; - -export const getPrefixColumns = (groupBy: DATAMAP_GROUPING) => { - let columnOrder: string[] = []; - if (DATAMAP_GROUPING.SYSTEM_DATA_USE === groupBy) { - columnOrder = [COLUMN_IDS.SYSTEM_NAME, COLUMN_IDS.DATA_USE]; - } - if (DATAMAP_GROUPING.DATA_USE_SYSTEM === groupBy) { - columnOrder = [COLUMN_IDS.DATA_USE, COLUMN_IDS.SYSTEM_NAME]; - } - return columnOrder; -}; From 483d7984e996af447e37e1bd4cc59d05af9821db Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Wed, 8 Jan 2025 11:49:18 -0700 Subject: [PATCH 03/50] Action Center results MVP (#5622) --- CHANGELOG.md | 2 +- .../admin-ui/cypress/e2e/action-center.cy.ts | 119 ++++++++++++ .../results/aggregate-results.json | 40 ++++ clients/admin-ui/cypress/support/stubs.ts | 13 ++ clients/admin-ui/package.json | 5 +- .../src/features/common/SearchBar.tsx | 2 +- .../admin-ui/src/features/common/api.slice.ts | 1 + .../src/features/common/nav/v2/nav-config.ts | 7 + .../src/features/common/nav/v2/routes.ts | 1 + .../common/table/v2/PaginationBar.tsx | 4 +- clients/admin-ui/src/features/common/utils.ts | 6 +- .../action-center/DisabledMonitorPage.tsx | 28 +++ .../action-center/EmptyMonitorResult.tsx | 15 ++ .../action-center/MonitorResult.tsx | 97 ++++++++++ .../action-center/actionCenter.slice.tsx | 24 +++ .../action-center/types.ts | 17 ++ .../features/locations/LocationManagement.tsx | 1 - .../locations/RegulationManagement.tsx | 1 - clients/admin-ui/src/flags.json | 6 + .../action-center/[monitorId]/index.tsx | 5 + .../data-discovery/action-center/index.tsx | 176 ++++++++++++++++++ clients/admin-ui/src/theme/global.scss | 14 ++ clients/fidesui/src/index.ts | 6 + clients/package-lock.json | 37 ++-- 24 files changed, 601 insertions(+), 26 deletions(-) create mode 100644 clients/admin-ui/cypress/e2e/action-center.cy.ts create mode 100644 clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts create mode 100644 clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx create mode 100644 clients/admin-ui/src/pages/data-discovery/action-center/index.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e7d248787..1862215a98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ## [Unreleased](https://github.com/ethyca/fides/compare/2.52.0...main) ### Added +- Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) ### Fixed @@ -29,7 +30,6 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) - Fixed issue where the Data Map report filter dialog was missing an Accordion item label [#5649](https://github.com/ethyca/fides/pull/5649) - ## [2.52.0](https://github.com/ethyca/fides/compare/2.51.2...2.52.0) ### Added diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts new file mode 100644 index 0000000000..b3541de4a7 --- /dev/null +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -0,0 +1,119 @@ +import { stubActionCenter, stubPlus } from "cypress/support/stubs"; + +import { + ACTION_CENTER_ROUTE, + INTEGRATION_MANAGEMENT_ROUTE, +} from "~/features/common/nav/v2/routes"; + +describe("Action center", () => { + beforeEach(() => { + cy.login(); + stubPlus(true); + stubActionCenter(); + }); + + describe("disabled web monitor", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/config*", { + body: { + detection_discovery: { + website_monitor_enabled: false, + }, + }, + }).as("getTranslationConfig"); + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should display a message that the web monitor is disabled", () => { + cy.wait("@getTranslationConfig"); + cy.contains("currently disabled").should("exist"); + }); + }); + + describe("empty action center", () => { + beforeEach(() => { + cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { + fixture: "empty-pagination.json", + }).as("getMonitorResults"); + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should display empty state", () => { + cy.wait("@getMonitorResults"); + cy.get("[data-testid='search-bar']").should("exist"); + cy.get(`[class*='ant-empty'] [class*='ant-empty-image']`).should("exist"); + cy.get( + `[class*='ant-empty'] a[href="${INTEGRATION_MANAGEMENT_ROUTE}"]`, + ).should("exist"); + }); + }); + + describe("Action center monitor results", () => { + const webMonitorKey = "my_web_monitor_2"; + const integrationMonitorKey = "My_New_BQ_Monitor"; + beforeEach(() => { + cy.visit(ACTION_CENTER_ROUTE); + }); + it("should render the current monitor results", () => { + cy.get("[data-testid='Action center']").should("exist"); + cy.wait("@getMonitorResults"); + cy.get("[data-testid*='monitor-result-']").should("have.length", 3); + cy.get("[data-testid^='monitor-result-']").each((result) => { + const monitorKey = result + .attr("data-testid") + .replace("monitor-result-", ""); + // linked title + cy.wrap(result) + .contains("assets detected") + .should("have.attr", "href", `${ACTION_CENTER_ROUTE}/${monitorKey}`); + // last monitored relative date with real date in tooltip + cy.wrap(result) + .find("[data-testid='monitor-date']") + .contains(" ago") + .realHover(); + cy.get(".ant-tooltip-inner").should("contain", "December"); + }); + // description + cy.getByTestId(`monitor-result-${webMonitorKey}`).should( + "contain", + "92 Browser Requests, 5 Cookies detected.", + ); + // monitor name + cy.getByTestId(`monitor-result-${webMonitorKey}`).should( + "contain", + "my web monitor 2", + ); + }); + it("should have appropriate actions for web monitors", () => { + cy.wait("@getMonitorResults"); + // Add button + // TODO: [HJ-337] uncomment when Add button is implemented + // cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); + // Review button + cy.getByTestId(`review-button-${webMonitorKey}`).should( + "have.attr", + "href", + `${ACTION_CENTER_ROUTE}/${webMonitorKey}`, + ); + }); + it.skip("Should have appropriate actions for Integrations monitors", () => { + cy.wait("@getMonitorResults"); + // Classify button + cy.getByTestId(`review-button-${integrationMonitorKey}`).should( + "have.attr", + "href", + `${ACTION_CENTER_ROUTE}/${integrationMonitorKey}`, + ); + // Ignore button + cy.getByTestId(`ignore-button-${integrationMonitorKey}`).should("exist"); + }); + it.skip("Should have appropriate actions for SSO monitors", () => { + cy.wait("@getMonitorResults"); + // Add button + cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); + // Ignore button + cy.getByTestId(`ignore-button-${webMonitorKey}`).should("exist"); + }); + it.skip("Should paginate results", () => { + // TODO: mock pagination and also test skeleton loading state + }); + }); +}); diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json new file mode 100644 index 0000000000..0a870f84e1 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json @@ -0,0 +1,40 @@ +{ + "items": [ + { + "name": "my web monitor 2", + "key": "my_web_monitor_2", + "last_monitored": "2024-12-17T17:31:20.791014Z", + "updates": { + "Browser Request": 92, + "Cookie": 5 + }, + "total_updates": 97 + }, + { + "name": "my web monitor 1", + "key": "my_web_monitor_1", + "last_monitored": "2024-12-17T17:31:02.319068Z", + "updates": { + "Browser Request": 201, + "Cookie": 24 + }, + "total_updates": 225 + }, + { + "name": "My New BQ Monitor", + "key": "My_New_BQ_Monitor", + "last_monitored": "2024-12-16T20:04:16.824025Z", + "updates": { + "Database": 2, + "Field": 216, + "Schema": 13, + "Table": 22 + }, + "total_updates": 253 + } + ], + "total": 3, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index bb2e40537c..2bc49dd917 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -502,3 +502,16 @@ export const stubFidesCloud = () => { domain_verification_records: [], }).as("getFidesCloud"); }; + +export const stubActionCenter = () => { + cy.intercept("GET", "/api/v1/config*", { + body: { + detection_discovery: { + website_monitor_enabled: true, + }, + }, + }).as("getTranslationConfig"); + cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { + fixture: "detection-discovery/results/aggregate-results", + }).as("getMonitorResults"); +}; diff --git a/clients/admin-ui/package.json b/clients/admin-ui/package.json index 1fd75f5903..34dd8bbb4d 100644 --- a/clients/admin-ui/package.json +++ b/clients/admin-ui/package.json @@ -30,6 +30,7 @@ }, "dependencies": { "@ant-design/cssinjs": "^1.21.0", + "@date-fns/tz": "^1.2.0", "@fontsource/inter": "^4.5.15", "@monaco-editor/react": "^4.6.0", "@reduxjs/toolkit": "^1.9.3", @@ -40,8 +41,8 @@ "cytoscape": "^3.30.0", "cytoscape-klay": "^3.1.4", "d3-hierarchy": "^3.1.2", - "date-fns": "^2.29.3", - "date-fns-tz": "^2.0.0", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", "eslint-plugin-tailwindcss": "^3.17.4", "fides-js": "^0.0.1", "fidesui": "*", diff --git a/clients/admin-ui/src/features/common/SearchBar.tsx b/clients/admin-ui/src/features/common/SearchBar.tsx index 973ca254ed..248d34e44b 100644 --- a/clients/admin-ui/src/features/common/SearchBar.tsx +++ b/clients/admin-ui/src/features/common/SearchBar.tsx @@ -24,7 +24,7 @@ const SearchBar = ({ onChange(event.target.value); return ( - + { const defaultPageIndex = 1; const [pageSize, setPageSize] = useState(PAGE_SIZES[0]); const [pageIndex, setPageIndex] = useState(defaultPageIndex); - const [totalPages, setTotalPages] = useState(); + const [totalPages, setTotalPages] = useState(1); const onPreviousPageClick = useCallback(() => { setPageIndex((prev) => prev - 1); }, [setPageIndex]); @@ -53,7 +53,7 @@ export const useServerSidePagination = () => { setPageIndex((prev) => prev + 1); }, [setPageIndex]); const isNextPageDisabled = useMemo( - () => pageIndex === totalPages, + () => !!totalPages && (pageIndex === totalPages || totalPages < 2), [pageIndex, totalPages], ); diff --git a/clients/admin-ui/src/features/common/utils.ts b/clients/admin-ui/src/features/common/utils.ts index acc4d86588..ab18bc6803 100644 --- a/clients/admin-ui/src/features/common/utils.ts +++ b/clients/admin-ui/src/features/common/utils.ts @@ -32,7 +32,7 @@ export const debounce = (fn: (props?: any) => void, ms = 0) => { }; export const formatDate = (value: string | number | Date): string => - format(new Date(value), "MMMM d, Y, KK:mm:ss z"); + format(new Date(value), "MMMM d, y, KK:mm:ss z"); export const utf8ToB64 = (str: string): string => window.btoa(unescape(encodeURIComponent(str))); @@ -116,3 +116,7 @@ export const getOptionsFromMap = ( label: value, value: key, })); + +export const getWebsiteIconUrl = (hostname: string) => { + return `https://icons.duckduckgo.com/ip3/${hostname}.ico`; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx new file mode 100644 index 0000000000..0cac2e4d62 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx @@ -0,0 +1,28 @@ +import { AntAlert as Alert, AntFlex as Flex, Spinner } from "fidesui"; + +import Layout from "~/features/common/Layout"; + +interface DisabledMonitorPageProps { + isConfigLoading: boolean; +} + +const DISABLED_MONITOR_MESSAGE = "Action center is currently disabled."; + +export const DisabledMonitorPage = ({ + isConfigLoading, +}: DisabledMonitorPageProps) => ( + + + {isConfigLoading ? ( + + ) : ( + + )} + + +); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx new file mode 100644 index 0000000000..f878a958b8 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx @@ -0,0 +1,15 @@ +import { AntButton as Button, AntEmpty as Empty } from "fidesui"; +import NextLink from "next/link"; + +import { INTEGRATION_MANAGEMENT_ROUTE } from "~/features/common/nav/v2/routes"; + +export const EmptyMonitorResult = () => ( + + + + + +); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx new file mode 100644 index 0000000000..24c8c49ac4 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/MonitorResult.tsx @@ -0,0 +1,97 @@ +import { formatDistance } from "date-fns"; +import { + AntAvatar as Avatar, + AntFlex as Flex, + AntList as List, + AntListItemProps as ListItemProps, + AntSkeleton as Skeleton, + AntTooltip as Tooltip, + AntTypography as Typography, + Icons, +} from "fidesui"; +import NextLink from "next/link"; + +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import { formatDate, getWebsiteIconUrl } from "~/features/common/utils"; + +import { MonitorSummary } from "./types"; + +const { Text } = Typography; + +interface MonitorResultProps extends ListItemProps { + monitorSummary: MonitorSummary; + showSkeleton?: boolean; +} + +export const MonitorResult = ({ + monitorSummary, + showSkeleton, + ...props +}: MonitorResultProps) => { + if (!monitorSummary) { + return null; + } + + const { + name, + property, + total_updates: totalUpdates, + updates, + last_monitored: lastMonitored, + warning, + key, + } = monitorSummary; + + const assetCountString = Object.entries(updates) + .map((update) => { + return `${update[1]} ${update[0]}s`; + }) + .join(", "); + + const lastMonitoredDistance = lastMonitored + ? formatDistance(new Date(lastMonitored), new Date(), { + addSuffix: true, + }) + : undefined; + + const iconUrl = property ? getWebsiteIconUrl(property) : undefined; + + return ( + + + } + title={ + + {`${totalUpdates} assets detected${property ? `on ${property}` : ""}`} + {!!warning && ( + + + + )} + + } + description={`${assetCountString} detected.`} + /> + + + {name} + + {!!lastMonitoredDistance && ( + + {lastMonitoredDistance} + + )} + + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx new file mode 100644 index 0000000000..6d217a0c4b --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx @@ -0,0 +1,24 @@ +import { baseApi } from "~/features/common/api.slice"; + +import { MonitorSummaryPaginatedResponse } from "./types"; + +const actionCenterApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getMonitorSummary: build.query< + MonitorSummaryPaginatedResponse, + { + pageIndex?: number; + pageSize?: number; + search?: string; + } + >({ + query: ({ pageIndex = 1, pageSize = 20, search }) => ({ + url: `/plus/discovery-monitor/aggregate-results`, + params: { page: pageIndex, size: pageSize, search }, + }), + providesTags: ["Monitor Summary"], + }), + }), +}); + +export const { useGetMonitorSummaryQuery } = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts new file mode 100644 index 0000000000..e33d824d58 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts @@ -0,0 +1,17 @@ +// TODO: [HJ-334] remove these in favor of autogenerated types from the API +export interface MonitorSummary { + updates: Record; + property?: string; + last_monitored: string | number; + key: string; + name: string; + total_updates: number; + warning?: boolean | string; +} + +export interface MonitorSummaryPaginatedResponse { + items: MonitorSummary[]; + page: number; + size: number; + total: number; +} diff --git a/clients/admin-ui/src/features/locations/LocationManagement.tsx b/clients/admin-ui/src/features/locations/LocationManagement.tsx index 40931434d1..ac244ebf8e 100644 --- a/clients/admin-ui/src/features/locations/LocationManagement.tsx +++ b/clients/admin-ui/src/features/locations/LocationManagement.tsx @@ -98,7 +98,6 @@ const LocationManagement = ({ data }: { data: LocationRegulationResponse }) => { placeholder="Search" search={search} onClear={() => setSearch("")} - data-testid="search-bar" /> diff --git a/clients/admin-ui/src/features/locations/RegulationManagement.tsx b/clients/admin-ui/src/features/locations/RegulationManagement.tsx index 795d2b77ff..980801e3fc 100644 --- a/clients/admin-ui/src/features/locations/RegulationManagement.tsx +++ b/clients/admin-ui/src/features/locations/RegulationManagement.tsx @@ -103,7 +103,6 @@ const RegulationManagement = ({ placeholder="Search" search={search} onClear={() => setSearch("")} - data-testid="search-bar" /> diff --git a/clients/admin-ui/src/flags.json b/clients/admin-ui/src/flags.json index f33a2e81ee..5f2b32cea2 100644 --- a/clients/admin-ui/src/flags.json +++ b/clients/admin-ui/src/flags.json @@ -36,6 +36,12 @@ "test": true, "production": false }, + "webMonitor": { + "description": "Monitor websites for activity", + "development": true, + "test": true, + "production": false + }, "ssoAuthentication": { "description": "SSO Authentication Providers (OpenID)", "development": true, diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx new file mode 100644 index 0000000000..6f07a74600 --- /dev/null +++ b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx @@ -0,0 +1,5 @@ +const MonitorResultSystems = () => { + return
Monitor Result Systems FPO
; +}; + +export default MonitorResultSystems; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx new file mode 100644 index 0000000000..6edb23321c --- /dev/null +++ b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx @@ -0,0 +1,176 @@ +import { + AntButton as Button, + AntDivider as Divider, + AntFlex as Flex, + AntList as List, + useToast, +} from "fidesui"; +import NextLink from "next/link"; +import { useCallback, useEffect, useState } from "react"; + +import Layout from "~/features/common/Layout"; +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + PaginationBar, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetMonitorSummaryQuery } from "~/features/data-discovery-and-detection/action-center/actionCenter.slice"; +import { DisabledMonitorPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorPage"; +import { EmptyMonitorResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorResult"; +import { MonitorResult } from "~/features/data-discovery-and-detection/action-center/MonitorResult"; +import { MonitorSummary } from "~/features/data-discovery-and-detection/action-center/types"; +import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput"; +import { useGetConfigurationSettingsQuery } from "~/features/privacy-requests"; + +const ActionCenterPage = () => { + const toast = useToast(); + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + const [searchQuery, setSearchQuery] = useState(""); + const { data: appConfig, isLoading: isConfigLoading } = + useGetConfigurationSettingsQuery({ + api_set: false, + }); + const webMonitorEnabled = + !!appConfig?.detection_discovery?.website_monitor_enabled; + + useEffect(() => { + resetPageIndexToDefault(); + }, [searchQuery, resetPageIndexToDefault]); + + const { data, isError, isLoading, isFetching } = useGetMonitorSummaryQuery( + { + pageIndex, + pageSize, + search: searchQuery, + }, + { skip: isConfigLoading || !webMonitorEnabled }, + ); + + useEffect(() => { + if (isError && !!toast && webMonitorEnabled) { + toast({ + title: "Error fetching data", + description: "Please try again later", + status: "error", + }); + } + }, [isError, toast, webMonitorEnabled]); + + useEffect(() => { + if (data) { + setTotalPages(data.total || 1); + } + }, [data, setTotalPages]); + + const results = data?.items || []; + const loadingResults = isFetching + ? (Array.from({ length: pageSize }, (_, index) => ({ + key: index.toString(), + updates: [], + last_monitored: null, + })) as any[]) + : []; + + // TODO: [HJ-337] Add button functionality + + // const handleAdd = (monidorId: string) => { + // console.log("Add report", monidorId); + // }; + + const getWebsiteMonitorActions = useCallback( + (monitorKey: string) => [ + // , + + + , + ], + [], + ); + + if (!webMonitorEnabled) { + return ; + } + + return ( + + + + + + + + , + }} + renderItem={(summary: MonitorSummary) => ( + + )} + /> + + {!!results && !!data?.total && data.total > pageSize && ( + <> + + + + )} + + ); +}; + +export default ActionCenterPage; diff --git a/clients/admin-ui/src/theme/global.scss b/clients/admin-ui/src/theme/global.scss index 5c7aa053e1..d4357518ce 100644 --- a/clients/admin-ui/src/theme/global.scss +++ b/clients/admin-ui/src/theme/global.scss @@ -1,5 +1,19 @@ @import "fidesui/src/palette/palette.module.scss"; +/** + * Chakra removes heading font weight, wheras Ant assumes browser defaults. + * This sets the font weight for headings back to the browser default for Ant support. + * Remove this once Chakra has been removed. + */ +h1, +h2, +h3, +h4, +h5, +h6 { + font-weight: bold; +} + /** * Adds the color variables from the palette to the root element */ diff --git a/clients/fidesui/src/index.ts b/clients/fidesui/src/index.ts index 6179fa6da0..25652dce60 100644 --- a/clients/fidesui/src/index.ts +++ b/clients/fidesui/src/index.ts @@ -11,6 +11,7 @@ export type { FlexProps as AntFlexProps, FormInstance as AntFormInstance, InputProps as AntInputProps, + ListProps as AntListProps, SelectProps as AntSelectProps, SwitchProps as AntSwitchProps, GetProps, @@ -18,19 +19,23 @@ export type { } from "antd/lib"; export { Alert as AntAlert, + Avatar as AntAvatar, Breadcrumb as AntBreadcrumb, Button as AntButton, Card as AntCard, Checkbox as AntCheckbox, Col as AntCol, Divider as AntDivider, + Empty as AntEmpty, Flex as AntFlex, Form as AntForm, Input as AntInput, Layout as AntLayout, + List as AntList, Menu as AntMenu, Radio as AntRadio, Row as AntRow, + Skeleton as AntSkeleton, Space as AntSpace, Switch as AntSwitch, Tag as AntTag, @@ -41,6 +46,7 @@ export type { BreadcrumbItemType as AntBreadcrumbItemType, BreadcrumbProps as AntBreadcrumbProps, } from "antd/lib/breadcrumb/Breadcrumb"; +export type { ListItemProps as AntListItemProps } from "antd/lib/list"; export type { BaseOptionType as AntBaseOptionType, DefaultOptionType as AntDefaultOptionType, diff --git a/clients/package-lock.json b/clients/package-lock.json index 62d4f02527..b3a00b04f1 100644 --- a/clients/package-lock.json +++ b/clients/package-lock.json @@ -18,6 +18,7 @@ "admin-ui": { "dependencies": { "@ant-design/cssinjs": "^1.21.0", + "@date-fns/tz": "^1.2.0", "@fontsource/inter": "^4.5.15", "@monaco-editor/react": "^4.6.0", "@reduxjs/toolkit": "^1.9.3", @@ -28,8 +29,8 @@ "cytoscape": "^3.30.0", "cytoscape-klay": "^3.1.4", "d3-hierarchy": "^3.1.2", - "date-fns": "^2.29.3", - "date-fns-tz": "^2.0.0", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", "eslint-plugin-tailwindcss": "^3.17.4", "fides-js": "^0.0.1", "fidesui": "*", @@ -3058,6 +3059,12 @@ "ms": "^2.1.1" } }, + "node_modules/@date-fns/tz": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@date-fns/tz/-/tz-1.2.0.tgz", + "integrity": "sha512-LBrd7MiJZ9McsOgxqWX7AaxrDjcFVjWH/tIKJd7pnR7McaslGYOP1QmmiBXdJH/H/yLCT+rcQ7FaPBUxRGUtrg==", + "license": "MIT" + }, "node_modules/@emotion/babel-plugin": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz", @@ -10328,26 +10335,22 @@ } }, "node_modules/date-fns": { - "version": "2.30.0", - "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", - "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", - "dependencies": { - "@babel/runtime": "^7.21.0" - }, - "engines": { - "node": ">=0.11" - }, + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz", + "integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==", + "license": "MIT", "funding": { - "type": "opencollective", - "url": "https://opencollective.com/date-fns" + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" } }, "node_modules/date-fns-tz": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-2.0.1.tgz", - "integrity": "sha512-fJCG3Pwx8HUoLhkepdsP7Z5RsucUi+ZBOxyM5d0ZZ6c4SdYustq0VMmOu6Wf7bli+yS/Jwp91TOCqn9jMcVrUA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-3.2.0.tgz", + "integrity": "sha512-sg8HqoTEulcbbbVXeg84u5UnlsQa8GS5QXMqjjYIhS4abEVVKIUwe0/l/UhrZdKaL/W5eWZNlbTeEIiOXTcsBQ==", + "license": "MIT", "peerDependencies": { - "date-fns": "2.x" + "date-fns": "^3.0.0 || ^4.0.0" } }, "node_modules/dayjs": { From a583252764b5507df81740882eba24a4a46ca898 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Thu, 9 Jan 2025 12:02:51 -0700 Subject: [PATCH 04/50] Action Center: View discovered System Aggregate Results (#5653) --- .../admin-ui/cypress/e2e/action-center.cy.ts | 58 +++++++++- .../aggregate-results.json | 0 .../system-aggregate-results.json | 84 +++++++++++++++ clients/admin-ui/cypress/support/stubs.ts | 9 +- .../admin-ui/src/features/common/api.slice.ts | 1 - .../features/common/custom-fields/Layout.tsx | 16 --- .../features/common/custom-fields/index.ts | 1 - ...nitorPage.tsx => DisabledMonitorsPage.tsx} | 10 +- ...itorResult.tsx => EmptyMonitorsResult.tsx} | 2 +- .../action-center/MonitorResult.tsx | 15 +-- .../action-center/action-center.slice.ts | 48 +++++++++ .../action-center/actionCenter.slice.tsx | 24 ----- .../useDiscoveredSystemAggregateColumns.tsx | 75 +++++++++++++ .../tables/DiscoveredSystemAggregateTable.tsx | 101 ++++++++++++++++++ .../DiscoveredSystemAggregateActionsCell.tsx | 14 +++ .../DiscoveredSystemAggregateStatusCell.tsx | 33 ++++++ .../action-center/types.ts | 24 +++-- .../ConnectionTypeLogo.tsx | 9 +- .../{types.ts => types.d.ts} | 11 -- .../action-center/[monitorId]/index.tsx | 26 ++++- .../data-discovery/action-center/index.tsx | 49 +++++---- .../src/types/common/PaginationQueryParams.ts | 8 ++ 22 files changed, 519 insertions(+), 99 deletions(-) rename clients/admin-ui/cypress/fixtures/detection-discovery/{results => activity-center}/aggregate-results.json (100%) create mode 100644 clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json delete mode 100644 clients/admin-ui/src/features/common/custom-fields/Layout.tsx rename clients/admin-ui/src/features/data-discovery-and-detection/action-center/{DisabledMonitorPage.tsx => DisabledMonitorsPage.tsx} (68%) rename clients/admin-ui/src/features/data-discovery-and-detection/action-center/{EmptyMonitorResult.tsx => EmptyMonitorsResult.tsx} (92%) create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts delete mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx rename clients/admin-ui/src/features/datastore-connections/{types.ts => types.d.ts} (91%) diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts index b3541de4a7..1f816ae3b3 100644 --- a/clients/admin-ui/cypress/e2e/action-center.cy.ts +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -46,7 +46,7 @@ describe("Action center", () => { }); }); - describe("Action center monitor results", () => { + describe("Action center monitor aggregate results", () => { const webMonitorKey = "my_web_monitor_2"; const integrationMonitorKey = "My_New_BQ_Monitor"; beforeEach(() => { @@ -116,4 +116,60 @@ describe("Action center", () => { // TODO: mock pagination and also test skeleton loading state }); }); + + describe("Action center system aggregate results", () => { + const webMonitorKey = "my_web_monitor_1"; + beforeEach(() => { + cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}`); + }); + it("should display a breadcrumb", () => { + cy.getByTestId("page-breadcrumb").within(() => { + cy.get("a.ant-breadcrumb-link") + .should("contain", "All activity") + .should("have.attr", "href", ACTION_CENTER_ROUTE); + cy.contains("my_web_monitor_1").should("exist"); + }); + }); + it("should render the aggregated system results in a table", () => { + cy.wait("@getSystemAggregateResults"); + cy.getByTestId("column-system_name").should("exist"); + cy.getByTestId("column-total_updates").should("exist"); + cy.getByTestId("column-data_use").should("exist"); + cy.getByTestId("column-locations").should("exist"); + cy.getByTestId("column-domains").should("exist"); + cy.getByTestId("column-actions").should("exist"); + cy.getByTestId("search-bar").should("exist"); + cy.getByTestId("pagination-btn").should("exist"); + cy.getByTestId("row-0-col-system_name").within(() => { + cy.getByTestId("change-icon").should("exist"); // new result + cy.contains("Uncategorized assets").should("exist"); + }); + // data use column should be empty for uncategorized assets + cy.getByTestId("row-0-col-data_use").children().should("have.length", 0); + cy.getByTestId("row-1-col-system_name").within(() => { + cy.getByTestId("change-icon").should("not.exist"); // existing result + cy.contains("Google Tag Manager").should("exist"); + }); + // TODO: data use column should not be empty for other assets + // cy.getByTestId("row-1-col-data_use").children().should("not.have.length", 0); + + // multiple locations + cy.getByTestId("row-2-col-locations").should("contain", "2 locations"); + // single location + cy.getByTestId("row-3-col-locations").should("contain", "USA"); + + // multiple domains + cy.getByTestId("row-0-col-domains").should("contain", "29 domains"); + // single domain + cy.getByTestId("row-3-col-domains").should( + "contain", + "analytics.google.com", + ); + }); + // it("should navigate to table view on row click", () => { + // cy.getByTestId("row-1").click(); + // cy.url().should("contain", "fds.1046"); + // cy.getByTestId("page-breadcrumb").should("contain", "fds.1046"); + // }); + }); }); diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json similarity index 100% rename from clients/admin-ui/cypress/fixtures/detection-discovery/results/aggregate-results.json rename to clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/aggregate-results.json diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json new file mode 100644 index 0000000000..3bcc4b2328 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json @@ -0,0 +1,84 @@ +{ + "items": [ + { + "id": null, + "name": null, + "system_key": null, + "vendor_id": null, + "total_updates": 108, + "locations": ["USA"], + "domains": [ + "alb.reddit.com", + "api.hubapi.com", + "app.revenuehero.io", + ".ethyca.com", + "ethyca.com", + "ethyca.fides-cdn.ethyca.com", + "forms.hscollectedforms.net", + "forms.hubspot.com", + "forms-na1.hsforms.com", + "googleads.g.doubleclick.net", + ".hsadspixel.net", + ".hsforms.com", + ".hs-scripts.com", + ".hubspot.com", + "js.hsadspixel.net", + "js.hs-analytics.net", + "js.hs-banner.com", + "js.hscollectedforms.net", + "js.hs-scripts.com", + "kit.fontawesome.com", + ".linkedin.com", + "pixel-config.reddit.com", + "px.ads.linkedin.com", + "snap.licdn.com", + "stats.g.doubleclick.net", + "track.hubspot.com", + "www.clickcease.com", + ".www.linkedin.com", + "www.redditstatic.com" + ] + }, + { + "id": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "name": "Google Tag Manager", + "system_key": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "vendor_id": "fds.1046", + "total_updates": 10, + "locations": ["USA"], + "domains": [ + "td.doubleclick.net", + "www.google.com", + "www.googletagmanager.com" + ] + }, + { + "id": "system_key-652c8984-ade7-470b-bce4-7e184621be9d", + "name": "Hubspot", + "system_key": "system_key-652c8984-ade7-470b-bce4-7e184621be9d", + "vendor_id": "fds.1053", + "total_updates": 6, + "locations": ["USA", "Canada"], + "domains": [ + "forms.hsforms.com", + ".hs-analytics.net", + ".hs-banner.com", + ".hsforms.net", + "js.hsforms.net" + ] + }, + { + "id": "fds.1047", + "name": "Google Analytics", + "system_key": null, + "vendor_id": "fds.1047", + "total_updates": 1, + "locations": ["USA"], + "domains": ["analytics.google.com"] + } + ], + "total": 4, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index 2bc49dd917..6c3109f1e9 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -512,6 +512,13 @@ export const stubActionCenter = () => { }, }).as("getTranslationConfig"); cy.intercept("GET", "/api/v1/plus/discovery-monitor/aggregate-results*", { - fixture: "detection-discovery/results/aggregate-results", + fixture: "detection-discovery/activity-center/aggregate-results", }).as("getMonitorResults"); + cy.intercept( + "GET", + "/api/v1//plus/discovery-monitor/system-aggregate-results*", + { + fixture: "detection-discovery/activity-center/system-aggregate-results", + }, + ).as("getSystemAggregateResults"); }; diff --git a/clients/admin-ui/src/features/common/api.slice.ts b/clients/admin-ui/src/features/common/api.slice.ts index 971ff2a766..4a6c5a9f1e 100644 --- a/clients/admin-ui/src/features/common/api.slice.ts +++ b/clients/admin-ui/src/features/common/api.slice.ts @@ -41,7 +41,6 @@ export const baseApi = createApi({ "Languages", "Locations", "Messaging Templates", - "Monitor Summary", "Dictionary", "System Vendors", "Latest Scan", diff --git a/clients/admin-ui/src/features/common/custom-fields/Layout.tsx b/clients/admin-ui/src/features/common/custom-fields/Layout.tsx deleted file mode 100644 index 6e3cb005f9..0000000000 --- a/clients/admin-ui/src/features/common/custom-fields/Layout.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { StackProps, VStack } from "fidesui"; -import * as React from "react"; - -const Layout = ({ children, ...props }: StackProps) => ( - - {children} - -); - -export { Layout }; diff --git a/clients/admin-ui/src/features/common/custom-fields/index.ts b/clients/admin-ui/src/features/common/custom-fields/index.ts index 90342f6b0e..c1f980165d 100644 --- a/clients/admin-ui/src/features/common/custom-fields/index.ts +++ b/clients/admin-ui/src/features/common/custom-fields/index.ts @@ -2,5 +2,4 @@ export * from "./constants"; export * from "./CustomFieldsList"; export * from "./helpers"; export * from "./hooks"; -export * from "./Layout"; export * from "./types"; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx similarity index 68% rename from clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx rename to clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx index 0cac2e4d62..cb80d6c6ba 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorPage.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/DisabledMonitorsPage.tsx @@ -2,15 +2,15 @@ import { AntAlert as Alert, AntFlex as Flex, Spinner } from "fidesui"; import Layout from "~/features/common/Layout"; -interface DisabledMonitorPageProps { +interface DisabledMonitorsPageProps { isConfigLoading: boolean; } -const DISABLED_MONITOR_MESSAGE = "Action center is currently disabled."; +const DISABLED_MONITORS_MESSAGE = "Action center is currently disabled."; -export const DisabledMonitorPage = ({ +export const DisabledMonitorsPage = ({ isConfigLoading, -}: DisabledMonitorPageProps) => ( +}: DisabledMonitorsPageProps) => ( {isConfigLoading ? ( @@ -18,7 +18,7 @@ export const DisabledMonitorPage = ({ ) : ( diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx similarity index 92% rename from clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx rename to clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx index f878a958b8..dfd82237fb 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorResult.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/EmptyMonitorsResult.tsx @@ -3,7 +3,7 @@ import NextLink from "next/link"; import { INTEGRATION_MANAGEMENT_ROUTE } from "~/features/common/nav/v2/routes"; -export const EmptyMonitorResult = () => ( +export const EmptyMonitorsResult = () => ( { - if (!monitorSummary) { - return null; - } + const [iconUrl, setIconUrl] = useState(undefined); const { name, @@ -54,7 +53,11 @@ export const MonitorResult = ({ }) : undefined; - const iconUrl = property ? getWebsiteIconUrl(property) : undefined; + useEffect(() => { + if (property) { + setIconUrl(getWebsiteIconUrl(property)); + } + }, [property]); return ( diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts new file mode 100644 index 0000000000..5e25656721 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/action-center.slice.ts @@ -0,0 +1,48 @@ +import { baseApi } from "~/features/common/api.slice"; +import { PaginationQueryParams } from "~/types/common/PaginationQueryParams"; + +import { + MonitorSummaryPaginatedResponse, + MonitorSystemAggregatePaginatedResponse, +} from "./types"; + +const actionCenterApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getAggregateMonitorResults: build.query< + MonitorSummaryPaginatedResponse, + { + search?: string; + } & PaginationQueryParams + >({ + query: ({ page = 1, size = 20, search }) => ({ + url: `/plus/discovery-monitor/aggregate-results`, + params: { page, size, search, diff_status: "addition" }, + }), + providesTags: ["Discovery Monitor Results"], + }), + getDiscoveredSystemAggregate: build.query< + MonitorSystemAggregatePaginatedResponse, + { + key: string; + search?: string; + } & PaginationQueryParams + >({ + query: ({ key, page = 1, size = 20, search }) => ({ + url: `/plus/discovery-monitor/system-aggregate-results`, + params: { + monitor_config_id: key, + page, + size, + search, + diff_status: "addition", + }, + }), + providesTags: ["Discovery Monitor Results"], + }), + }), +}); + +export const { + useGetAggregateMonitorResultsQuery, + useGetDiscoveredSystemAggregateQuery, +} = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx deleted file mode 100644 index 6d217a0c4b..0000000000 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/actionCenter.slice.tsx +++ /dev/null @@ -1,24 +0,0 @@ -import { baseApi } from "~/features/common/api.slice"; - -import { MonitorSummaryPaginatedResponse } from "./types"; - -const actionCenterApi = baseApi.injectEndpoints({ - endpoints: (build) => ({ - getMonitorSummary: build.query< - MonitorSummaryPaginatedResponse, - { - pageIndex?: number; - pageSize?: number; - search?: string; - } - >({ - query: ({ pageIndex = 1, pageSize = 20, search }) => ({ - url: `/plus/discovery-monitor/aggregate-results`, - params: { page: pageIndex, size: pageSize, search }, - }), - providesTags: ["Monitor Summary"], - }), - }), -}); - -export const { useGetMonitorSummaryQuery } = actionCenterApi; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx new file mode 100644 index 0000000000..747ab1ef6c --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredSystemAggregateColumns.tsx @@ -0,0 +1,75 @@ +import { createColumnHelper } from "@tanstack/react-table"; + +import { DefaultCell } from "~/features/common/table/v2"; + +import { DiscoveredSystemActionsCell } from "../tables/cells/DiscoveredSystemAggregateActionsCell"; +import { DiscoveredSystemStatusCell } from "../tables/cells/DiscoveredSystemAggregateStatusCell"; +import { MonitorSystemAggregate } from "../types"; + +export const useDiscoveredSystemAggregateColumns = () => { + const columnHelper = createColumnHelper(); + + const columns = [ + columnHelper.accessor((row) => row.name, { + id: "system_name", + cell: (props) => ( + + ), + header: "System", + meta: { + width: "auto", + }, + }), + columnHelper.accessor((row) => row.total_updates, { + id: "total_updates", + cell: (props) => , + header: "Assets", + size: 80, + }), + columnHelper.display({ + id: "data_use", + header: "Categories of consent", + meta: { + width: "auto", + }, + }), + columnHelper.accessor((row) => row.locations, { + id: "locations", + cell: (props) => ( + 1 + ? `${props.getValue().length} locations` + : props.getValue()[0] + } + /> + ), + header: "Locations", + }), + columnHelper.accessor((row) => row.domains, { + id: "domains", + cell: (props) => ( + 1 + ? `${props.getValue().length} domains` + : props.getValue()[0] + } + /> + ), + header: "Domains", + }), + columnHelper.display({ + id: "actions", + cell: (props) => ( + + ), + header: "Actions", + meta: { + width: "auto", + }, + }), + ]; + + return { columns }; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx new file mode 100644 index 0000000000..01d4b8c4d5 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable.tsx @@ -0,0 +1,101 @@ +import { getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { Box, Flex } from "fidesui"; +import { useEffect, useState } from "react"; + +import { + FidesTableV2, + PaginationBar, + TableActionBar, + TableSkeletonLoader, + useServerSidePagination, +} from "~/features/common/table/v2"; +import { useGetDiscoveredSystemAggregateQuery } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; + +import { SearchInput } from "../../SearchInput"; +import { useDiscoveredSystemAggregateColumns } from "../hooks/useDiscoveredSystemAggregateColumns"; + +interface DiscoveredSystemAggregateTableProps { + monitorId: string; +} + +export const DiscoveredSystemAggregateTable = ({ + monitorId, +}: DiscoveredSystemAggregateTableProps) => { + const { + PAGE_SIZES, + pageSize, + setPageSize, + onPreviousPageClick, + isPreviousPageDisabled, + onNextPageClick, + isNextPageDisabled, + startRange, + endRange, + pageIndex, + setTotalPages, + resetPageIndexToDefault, + } = useServerSidePagination(); + const [searchQuery, setSearchQuery] = useState(""); + + useEffect(() => { + resetPageIndexToDefault(); + }, [monitorId, searchQuery, resetPageIndexToDefault]); + + const { data, isLoading, isFetching } = useGetDiscoveredSystemAggregateQuery({ + key: monitorId, + page: pageIndex, + size: pageSize, + search: searchQuery, + }); + + useEffect(() => { + if (data) { + setTotalPages(data.pages || 1); + } + }, [data, setTotalPages]); + + const { columns } = useDiscoveredSystemAggregateColumns(); + + const tableInstance = useReactTable({ + getCoreRowModel: getCoreRowModel(), + columns, + manualPagination: true, + data: data?.items || [], + columnResizeMode: "onChange", + }); + + if (isLoading) { + return ; + } + + return ( + <> + + + + + + + + + + + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx new file mode 100644 index 0000000000..d00127b884 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateActionsCell.tsx @@ -0,0 +1,14 @@ +import { AntFlex as Flex } from "fidesui"; + +import { MonitorSystemAggregate } from "../../types"; + +interface DiscoveredSystemActionsCellProps { + system: MonitorSystemAggregate; +} + +export const DiscoveredSystemActionsCell = ({ + system, +}: DiscoveredSystemActionsCellProps) => { + console.log(system); + return ; +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx new file mode 100644 index 0000000000..3c13f34290 --- /dev/null +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredSystemAggregateStatusCell.tsx @@ -0,0 +1,33 @@ +import { Flex, Text, Tooltip } from "fidesui"; + +import { STATUS_INDICATOR_MAP } from "~/features/data-discovery-and-detection/statusIndicators"; + +import { MonitorSystemAggregate } from "../../types"; + +interface DiscoveredSystemStatusCellProps { + system: MonitorSystemAggregate; +} + +export const DiscoveredSystemStatusCell = ({ + system, +}: DiscoveredSystemStatusCellProps) => { + return ( + + {!system?.system_key && ( + + {/* icon has to be wrapped in a span for the tooltip to work */} + {STATUS_INDICATOR_MAP.Change} + + )} + + {system?.name || "Uncategorized assets"} + + + ); +}; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts index e33d824d58..f2933bca51 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts +++ b/clients/admin-ui/src/features/data-discovery-and-detection/action-center/types.ts @@ -1,7 +1,9 @@ +import { PaginatedResponse } from "~/types/common/PaginationQueryParams"; + // TODO: [HJ-334] remove these in favor of autogenerated types from the API -export interface MonitorSummary { +export interface MonitorAggregatedResults { updates: Record; - property?: string; + property?: string; // this is a guess, it doesn't exist yet in the API last_monitored: string | number; key: string; name: string; @@ -9,9 +11,17 @@ export interface MonitorSummary { warning?: boolean | string; } -export interface MonitorSummaryPaginatedResponse { - items: MonitorSummary[]; - page: number; - size: number; - total: number; +export interface MonitorSummaryPaginatedResponse + extends PaginatedResponse {} + +export interface MonitorSystemAggregate { + name: string; + system_key: string | null; // null when the system is not a known system + vendor_id: string; + total_updates: 0; + locations: string[]; + domains: string[]; } + +export interface MonitorSystemAggregatePaginatedResponse + extends PaginatedResponse {} diff --git a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx index fc51a7ae17..86b15a4645 100644 --- a/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx +++ b/clients/admin-ui/src/features/datastore-connections/ConnectionTypeLogo.tsx @@ -12,12 +12,19 @@ import { CONNECTOR_LOGOS_PATH, FALLBACK_CONNECTOR_LOGOS_PATH, } from "./constants"; -import { isConnectionSystemTypeMap, isDatastoreConnection } from "./types"; type ConnectionTypeLogoProps = { data: string | ConnectionConfigurationResponse | ConnectionSystemTypeMap; }; +const isDatastoreConnection = ( + obj: any, +): obj is ConnectionConfigurationResponse => + (obj as ConnectionConfigurationResponse).connection_type !== undefined; + +const isConnectionSystemTypeMap = (obj: any): obj is ConnectionSystemTypeMap => + (obj as ConnectionSystemTypeMap).encoded_icon !== undefined; + const ConnectionTypeLogo = ({ data, ...props diff --git a/clients/admin-ui/src/features/datastore-connections/types.ts b/clients/admin-ui/src/features/datastore-connections/types.d.ts similarity index 91% rename from clients/admin-ui/src/features/datastore-connections/types.ts rename to clients/admin-ui/src/features/datastore-connections/types.d.ts index e4a171038f..7d1e20d841 100644 --- a/clients/admin-ui/src/features/datastore-connections/types.ts +++ b/clients/admin-ui/src/features/datastore-connections/types.d.ts @@ -1,6 +1,5 @@ import { ConnectionConfigurationResponse, - ConnectionSystemTypeMap, ConnectionType, DatasetConfigCtlDataset, SystemType, @@ -128,16 +127,6 @@ export type DatastoreConnectionResponse = { ]; }; -export const isDatastoreConnection = ( - obj: any, -): obj is ConnectionConfigurationResponse => - (obj as ConnectionConfigurationResponse).connection_type !== undefined; - -export const isConnectionSystemTypeMap = ( - obj: any, -): obj is ConnectionSystemTypeMap => - (obj as ConnectionSystemTypeMap).encoded_icon !== undefined; - export type DatastoreConnectionParams = { search: string; connection_type?: string[]; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx index 6f07a74600..5f96ddfa1a 100644 --- a/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx +++ b/clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/index.tsx @@ -1,5 +1,27 @@ -const MonitorResultSystems = () => { - return
Monitor Result Systems FPO
; +import { NextPage } from "next"; +import { useRouter } from "next/router"; + +import FixedLayout from "~/features/common/FixedLayout"; +import { ACTION_CENTER_ROUTE } from "~/features/common/nav/v2/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { DiscoveredSystemAggregateTable } from "~/features/data-discovery-and-detection/action-center/tables/DiscoveredSystemAggregateTable"; + +const MonitorResultSystems: NextPage = () => { + const router = useRouter(); + const monitorId = decodeURIComponent(router.query.monitorId as string); + + return ( + + + + + ); }; export default MonitorResultSystems; diff --git a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx index 6edb23321c..598a357f4d 100644 --- a/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx +++ b/clients/admin-ui/src/pages/data-discovery/action-center/index.tsx @@ -15,11 +15,11 @@ import { PaginationBar, useServerSidePagination, } from "~/features/common/table/v2"; -import { useGetMonitorSummaryQuery } from "~/features/data-discovery-and-detection/action-center/actionCenter.slice"; -import { DisabledMonitorPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorPage"; -import { EmptyMonitorResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorResult"; +import { useGetAggregateMonitorResultsQuery } from "~/features/data-discovery-and-detection/action-center/action-center.slice"; +import { DisabledMonitorsPage } from "~/features/data-discovery-and-detection/action-center/DisabledMonitorsPage"; +import { EmptyMonitorsResult } from "~/features/data-discovery-and-detection/action-center/EmptyMonitorsResult"; import { MonitorResult } from "~/features/data-discovery-and-detection/action-center/MonitorResult"; -import { MonitorSummary } from "~/features/data-discovery-and-detection/action-center/types"; +import { MonitorAggregatedResults } from "~/features/data-discovery-and-detection/action-center/types"; import { SearchInput } from "~/features/data-discovery-and-detection/SearchInput"; import { useGetConfigurationSettingsQuery } from "~/features/privacy-requests"; @@ -51,14 +51,15 @@ const ActionCenterPage = () => { resetPageIndexToDefault(); }, [searchQuery, resetPageIndexToDefault]); - const { data, isError, isLoading, isFetching } = useGetMonitorSummaryQuery( - { - pageIndex, - pageSize, - search: searchQuery, - }, - { skip: isConfigLoading || !webMonitorEnabled }, - ); + const { data, isError, isLoading, isFetching } = + useGetAggregateMonitorResultsQuery( + { + page: pageIndex, + size: pageSize, + search: searchQuery, + }, + { skip: isConfigLoading || !webMonitorEnabled }, + ); useEffect(() => { if (isError && !!toast && webMonitorEnabled) { @@ -123,7 +124,7 @@ const ActionCenterPage = () => { ); if (!webMonitorEnabled) { - return ; + return ; } return ( @@ -141,16 +142,20 @@ const ActionCenterPage = () => { loading={isLoading} dataSource={results || loadingResults} locale={{ - emptyText: , + emptyText: , + }} + renderItem={(summary: MonitorAggregatedResults) => { + return ( + !!summary && ( + + ) + ); }} - renderItem={(summary: MonitorSummary) => ( - - )} /> {!!results && !!data?.total && data.total > pageSize && ( diff --git a/clients/admin-ui/src/types/common/PaginationQueryParams.ts b/clients/admin-ui/src/types/common/PaginationQueryParams.ts index bfdeda1475..0600daee7f 100644 --- a/clients/admin-ui/src/types/common/PaginationQueryParams.ts +++ b/clients/admin-ui/src/types/common/PaginationQueryParams.ts @@ -2,3 +2,11 @@ export interface PaginationQueryParams { page: number; size: number; } + +export interface PaginatedResponse { + items: T[]; + page: number; + size: number; + total: number; + pages: number; +} From 1de84b5837eb68474c3325abbcf59219dfea69d4 Mon Sep 17 00:00:00 2001 From: Andres Torres Date: Fri, 10 Jan 2025 08:17:38 -0600 Subject: [PATCH 05/50] HJ-352 - Fixes some fields that are not present on the CSV and they shouldn't be present on the web datamap report (#5645) Co-authored-by: Jason Gill --- .../custom-reports/custom-report.json | 5 +- .../cypress/fixtures/datamap/datamap.json | 18 ------ .../fixtures/datamap/empty_datamap.json | 2 - .../cypress/fixtures/datamap/minimal.json | 60 ------------------- .../src/features/datamap/datamap.slice.ts | 7 +-- .../datamap/reporting/DatamapReportTable.tsx | 28 +++++---- .../reporting/DatamapReportTableColumns.tsx | 24 +------- .../features/datamap/reporting/constants.tsx | 15 ++--- .../src/types/api/models/DatamapReport.ts | 3 - .../src/components/types/api/models/System.ts | 4 -- .../types/api/models/DatamapReport.ts | 3 - 11 files changed, 26 insertions(+), 143 deletions(-) diff --git a/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json b/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json index b2e2238323..9a3ae9bc4e 100644 --- a/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json +++ b/clients/admin-ui/cypress/fixtures/custom-reports/custom-report.json @@ -24,7 +24,7 @@ "legal_address", "cookie_refresh", "data_security_practices", - "DATA_SHARED_WITH_THIRD_PARTIES", + "data_shared_with_third_parties", "data_stewards", "declaration_name", "does_international_transfers", @@ -40,7 +40,6 @@ "legal_basis_for_profiling", "legal_basis_for_transfers", "legitimate_interest_disclosure_url", - "link_to_processor_contract", "processes_personal_data", "reason_for_exemption", "requires_data_protection_assessments", @@ -48,8 +47,6 @@ "retention_period", "shared_categories", "special_category_legal_basis", - "system_dependencies", - "third_country_safeguards", "third_parties", "system_undeclared_data_categories", "data_use_undeclared_data_categories", diff --git a/clients/admin-ui/cypress/fixtures/datamap/datamap.json b/clients/admin-ui/cypress/fixtures/datamap/datamap.json index fa73ee31ca..f4607dfe6d 100644 --- a/clients/admin-ui/cypress/fixtures/datamap/datamap.json +++ b/clients/admin-ui/cypress/fixtures/datamap/datamap.json @@ -6,9 +6,7 @@ "system.privacy_declaration.data_use.name": "Purpose of Processing", "system.privacy_declaration.data_subjects.name": "Categories of Individuals", "unioned_data_categories": "Categories of Personal Data (Fides Taxonomy)", - "system.link_to_processor_contract": "Link to Contract with Processor", "third_country_combined": "Third Country Transfers", - "system.third_country_safeguards": "Safeguards for Exceptional Transfers of Personal Data", "organization.link_to_security_policy": "General Description of Security Measures", "system.privacy_declaration.data_subjects.rights_available": "Rights available to individuals", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "Existence of automated decision-making, including profiling (if applicable)", @@ -24,9 +22,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.contact", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -42,9 +38,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.device.cookie_id", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -60,9 +54,7 @@ "system.privacy_declaration.data_use.name": "Advertising, Marketing or Promotion", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.device.cookie_id", - "system.link_to_processor_contract": "", "third_country_combined": "N/A", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -78,9 +70,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.contact.email", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -96,9 +86,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "system.operations", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -114,9 +102,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.unique_id", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -132,9 +118,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.contact.address.state", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", @@ -150,9 +134,7 @@ "system.privacy_declaration.data_use.name": "System", "system.privacy_declaration.data_subjects.name": "Customer", "unioned_data_categories": "user.name", - "system.link_to_processor_contract": "", "third_country_combined": "USA, GBR, CAN", - "system.third_country_safeguards": "", "organization.link_to_security_policy": "https://ethyca.com/privacy-policy/", "system.privacy_declaration.data_subjects.rights_available": "No data subject rights listed", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "N/A", diff --git a/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json b/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json index db645b12be..47ab0e62cc 100644 --- a/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json +++ b/clients/admin-ui/cypress/fixtures/datamap/empty_datamap.json @@ -6,9 +6,7 @@ "system.privacy_declaration.data_use.name": "Purpose of Processing", "system.privacy_declaration.data_subjects.name": "Categories of Individuals", "unioned_data_categories": "Categories of Personal Data (Fides Taxonomy)", - "system.link_to_processor_contract": "Link to Contract with Processor", "third_country_combined": "Third Country Transfers", - "system.third_country_safeguards": "Safeguards for Exceptional Transfers of Personal Data", "organization.link_to_security_policy": "General Description of Security Measures", "system.privacy_declaration.data_subjects.rights_available": "Rights available to individuals", "system.privacy_declaration.data_subjects.automated_decisions_or_profiling": "Existence of automated decision-making, including profiling (if applicable)", diff --git a/clients/admin-ui/cypress/fixtures/datamap/minimal.json b/clients/admin-ui/cypress/fixtures/datamap/minimal.json index 2b3c178ba5..d0eea25ece 100644 --- a/clients/admin-ui/cypress/fixtures/datamap/minimal.json +++ b/clients/admin-ui/cypress/fixtures/datamap/minimal.json @@ -33,7 +33,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -42,13 +41,11 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [ "user.contact.email", "user.device.cookie_id" ], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -88,7 +85,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -97,10 +93,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -140,7 +134,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -149,10 +142,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -192,7 +183,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -201,10 +191,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -244,7 +232,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -253,10 +240,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -296,7 +281,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -305,10 +289,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -348,7 +330,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -357,10 +338,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -400,7 +379,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -409,10 +387,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -452,7 +428,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -461,10 +436,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -504,7 +477,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -513,10 +485,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -556,7 +526,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -565,10 +534,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -608,7 +575,6 @@ "legal_basis_for_transfers": [], "legal_name": "1Agency", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://www.1agency.de/datenschutz", "processes_personal_data": true, "reason_for_exemption": null, @@ -617,10 +583,8 @@ "retention_period": null, "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1Agency", "system_undeclared_data_categories": [], - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": false, @@ -677,7 +641,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -686,9 +649,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -745,7 +706,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -754,9 +714,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -813,7 +771,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -822,9 +779,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -881,7 +836,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -890,9 +844,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -949,7 +901,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -958,9 +909,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -1017,7 +966,6 @@ "legal_basis_for_transfers": [], "legal_name": "1plusX AG", "legitimate_interest_disclosure_url": "https://www.1plusx.com/privacy-policy/", - "link_to_processor_contract": null, "privacy_policy": "https://www.1plusx.com/privacy-policy/", "processes_personal_data": true, "reason_for_exemption": null, @@ -1026,9 +974,7 @@ "retention_period": "90", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1plusX", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": true, "uses_non_cookie_access": true, @@ -1067,7 +1013,6 @@ "legal_basis_for_transfers": [], "legal_name": "1trn", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": "https://1trn.com/privacy.php", "processes_personal_data": true, "reason_for_exemption": null, @@ -1076,9 +1021,7 @@ "retention_period": null, "shared_categories": null, "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "1trn", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": false, "uses_non_cookie_access": false, @@ -1117,7 +1060,6 @@ "legal_basis_for_transfers": [], "legal_name": "", "legitimate_interest_disclosure_url": null, - "link_to_processor_contract": null, "privacy_policy": null, "processes_personal_data": true, "reason_for_exemption": null, @@ -1126,9 +1068,7 @@ "retention_period": "", "shared_categories": [], "special_category_legal_basis": null, - "system_dependencies": null, "system_name": "test", - "third_country_safeguards": null, "third_parties": null, "uses_cookies": false, "uses_non_cookie_access": false, diff --git a/clients/admin-ui/src/features/datamap/datamap.slice.ts b/clients/admin-ui/src/features/datamap/datamap.slice.ts index a383763127..72a1721c10 100644 --- a/clients/admin-ui/src/features/datamap/datamap.slice.ts +++ b/clients/admin-ui/src/features/datamap/datamap.slice.ts @@ -46,12 +46,7 @@ const DEFAULT_ACTIVE_COLUMNS = [ SYSTEM_DESCRIPTION, ]; -const DEPRECATED_COLUMNS = [ - "third_country_combined", - "system.third_country_safeguards", - "dataset.fides_key", - "system.link_to_processor_contract", -]; +const DEPRECATED_COLUMNS = ["third_country_combined", "dataset.fides_key"]; // API endpoints const datamapApi = baseApi.injectEndpoints({ diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx index d621bb1bf0..7960003dda 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx @@ -173,7 +173,7 @@ export const DatamapReportTable = () => { const [ exportMinimalDatamapReport, - { isLoading: isExportingReport, isSuccess: isExportReportSuccess }, + { isLoading: isExportingReport, isError: isExportReportError }, ] = useExportMinimalDatamapReportMutation(); const { data, totalRows } = useMemo(() => { @@ -208,15 +208,17 @@ export const DatamapReportTable = () => { const columns = useMemo( () => - getDatamapReportColumns({ - onSelectRow: (row) => setSelectedSystemId(row.fides_key), - getDataUseDisplayName, - getDataCategoryDisplayName, - getDataSubjectDisplayName, - datamapReport, - customFields, - isRenaming: isRenamingColumns, - }), + datamapReport + ? getDatamapReportColumns({ + onSelectRow: (row) => setSelectedSystemId(row.fides_key), + getDataUseDisplayName, + getDataCategoryDisplayName, + getDataSubjectDisplayName, + datamapReport, + customFields, + isRenaming: isRenamingColumns, + }) + : [], [ getDataUseDisplayName, getDataSubjectDisplayName, @@ -274,7 +276,7 @@ export const DatamapReportTable = () => { }, }, }).then(() => { - if (isExportReportSuccess) { + if (!isExportReportError) { onExportReportClose(); } }); @@ -303,7 +305,7 @@ export const DatamapReportTable = () => { }); useEffect(() => { - if (groupBy && !!tableInstance) { + if (groupBy && !!tableInstance && !!datamapReport) { if (tableInstance.getState().columnOrder.length === 0) { const tableColumnIds = tableInstance.getAllColumns().map((c) => c.id); setColumnOrder(getColumnOrder(groupBy, tableColumnIds)); @@ -314,7 +316,7 @@ export const DatamapReportTable = () => { } } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [groupBy, tableInstance]); + }, [groupBy, tableInstance, datamapReport]); useEffect(() => { // changing the groupBy should wait until the data is loaded to update the grouping diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx index b8d2ffda86..0325c26b88 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTableColumns.tsx @@ -241,6 +241,9 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.data_shared_with_third_parties, { id: COLUMN_IDS.DATA_SHARED_WITH_THIRD_PARTIES, }), + columnHelper.accessor((row) => row.processes_special_category_data, { + id: COLUMN_IDS.PROCESSES_SPECIAL_CATEGORY_DATA, + }), columnHelper.accessor((row) => row.data_stewards, { id: COLUMN_IDS.DATA_STEWARDS, cell: (props) => ( @@ -364,9 +367,6 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.legitimate_interest_disclosure_url, { id: COLUMN_IDS.LEGITIMATE_INTEREST_DISCLOSURE_URL, }), - columnHelper.accessor((row) => row.link_to_processor_contract, { - id: COLUMN_IDS.LINK_TO_PROCESSOR_CONTRACT, - }), columnHelper.accessor((row) => row.processes_personal_data, { id: COLUMN_IDS.PROCESSES_PERSONAL_DATA, }), @@ -414,24 +414,6 @@ export const getDatamapReportColumns = ({ columnHelper.accessor((row) => row.special_category_legal_basis, { id: COLUMN_IDS.SPECIAL_CATEGORY_LEGAL_BASIS, }), - columnHelper.accessor((row) => row.system_dependencies, { - id: COLUMN_IDS.SYSTEM_DEPENDENCIES, - cell: (props) => ( - - ), - meta: { - showHeaderMenu: !isRenaming, - width: "auto", - }, - }), - columnHelper.accessor((row) => row.third_country_safeguards, { - id: COLUMN_IDS.THIRD_COUNTRY_SAFEGUARDS, - }), columnHelper.accessor((row) => row.third_parties, { id: COLUMN_IDS.THIRD_PARTIES, }), diff --git a/clients/admin-ui/src/features/datamap/reporting/constants.tsx b/clients/admin-ui/src/features/datamap/reporting/constants.tsx index 8ab825e1f4..e471370b60 100644 --- a/clients/admin-ui/src/features/datamap/reporting/constants.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/constants.tsx @@ -27,7 +27,7 @@ export enum COLUMN_IDS { LEGAL_ADDRESS = "legal_address", COOKIE_REFRESH = "cookie_refresh", DATA_SECURITY_PRACTICES = "data_security_practices", - DATA_SHARED_WITH_THIRD_PARTIES = "DATA_SHARED_WITH_THIRD_PARTIES", + DATA_SHARED_WITH_THIRD_PARTIES = "data_shared_with_third_parties", DATA_STEWARDS = "data_stewards", DECLARATION_NAME = "declaration_name", DOES_INTERNATIONAL_TRANSFERS = "does_international_transfers", @@ -43,7 +43,6 @@ export enum COLUMN_IDS { LEGAL_BASIS_FOR_PROFILING = "legal_basis_for_profiling", LEGAL_BASIS_FOR_TRANSFERS = "legal_basis_for_transfers", LEGITIMATE_INTEREST_DISCLOSURE_URL = "legitimate_interest_disclosure_url", - LINK_TO_PROCESSOR_CONTRACT = "link_to_processor_contract", PROCESSES_PERSONAL_DATA = "processes_personal_data", REASON_FOR_EXEMPTION = "reason_for_exemption", REQUIRES_DATA_PROTECTION_ASSESSMENTS = "requires_data_protection_assessments", @@ -51,8 +50,6 @@ export enum COLUMN_IDS { RETENTION_PERIOD = "retention_period", SHARED_CATEGORIES = "shared_categories", SPECIAL_CATEGORY_LEGAL_BASIS = "special_category_legal_basis", - SYSTEM_DEPENDENCIES = "system_dependencies", - THIRD_COUNTRY_SAFEGUARDS = "third_country_safeguards", THIRD_PARTIES = "third_parties", COOKIES = "cookies", USES_COOKIES = "uses_cookies", @@ -60,6 +57,7 @@ export enum COLUMN_IDS { USES_PROFILING = "uses_profiling", SYSTEM_UNDECLARED_DATA_CATEGORIES = "system_undeclared_data_categories", DATA_USE_UNDECLARED_DATA_CATEGORIES = "data_use_undeclared_data_categories", + PROCESSES_SPECIAL_CATEGORY_DATA = "processes_special_category_data", } export const DEFAULT_COLUMN_NAMES: Record = { @@ -81,7 +79,7 @@ export const DEFAULT_COLUMN_NAMES: Record = { [COLUMN_IDS.DECLARATION_NAME]: "Declaration name", [COLUMN_IDS.DOES_INTERNATIONAL_TRANSFERS]: "Does international transfers", [COLUMN_IDS.DPA_LOCATION]: "DPA location", - [COLUMN_IDS.DESTINATIONS]: "Destinations", + [COLUMN_IDS.DESTINATIONS]: "Destination", [COLUMN_IDS.EXEMPT_FROM_PRIVACY_REGULATIONS]: "Exempt from privacy regulations", [COLUMN_IDS.FEATURES]: "Features", @@ -89,13 +87,12 @@ export const DEFAULT_COLUMN_NAMES: Record = { [COLUMN_IDS.FLEXIBLE_LEGAL_BASIS_FOR_PROCESSING]: "Flexible legal basis for processing", [COLUMN_IDS.IMPACT_ASSESSMENT_LOCATION]: "Impact assessment location", - [COLUMN_IDS.SOURCES]: "Sources", + [COLUMN_IDS.SOURCES]: "Source", [COLUMN_IDS.JOINT_CONTROLLER_INFO]: "Joint controller info", [COLUMN_IDS.LEGAL_BASIS_FOR_PROFILING]: "Legal basis for profiling", [COLUMN_IDS.LEGAL_BASIS_FOR_TRANSFERS]: "Legal basis for transfers", [COLUMN_IDS.LEGITIMATE_INTEREST_DISCLOSURE_URL]: "Legitimate interest disclosure URL", - [COLUMN_IDS.LINK_TO_PROCESSOR_CONTRACT]: "Link to processor contract", [COLUMN_IDS.PROCESSES_PERSONAL_DATA]: "Processes personal data", [COLUMN_IDS.REASON_FOR_EXEMPTION]: "Reason for exemption", [COLUMN_IDS.REQUIRES_DATA_PROTECTION_ASSESSMENTS]: @@ -104,8 +101,6 @@ export const DEFAULT_COLUMN_NAMES: Record = { [COLUMN_IDS.RETENTION_PERIOD]: "Retention period", [COLUMN_IDS.SHARED_CATEGORIES]: "Shared categories", [COLUMN_IDS.SPECIAL_CATEGORY_LEGAL_BASIS]: "Special category legal basis", - [COLUMN_IDS.SYSTEM_DEPENDENCIES]: "System dependencies", - [COLUMN_IDS.THIRD_COUNTRY_SAFEGUARDS]: "Third country safeguards", [COLUMN_IDS.THIRD_PARTIES]: "Third parties", [COLUMN_IDS.SYSTEM_UNDECLARED_DATA_CATEGORIES]: "System undeclared data categories", @@ -115,4 +110,6 @@ export const DEFAULT_COLUMN_NAMES: Record = { [COLUMN_IDS.USES_COOKIES]: "Uses cookies", [COLUMN_IDS.USES_NON_COOKIE_ACCESS]: "Uses non-cookie access", [COLUMN_IDS.USES_PROFILING]: "Uses profiling", + [COLUMN_IDS.PROCESSES_SPECIAL_CATEGORY_DATA]: + "Processes special category data", }; diff --git a/clients/admin-ui/src/types/api/models/DatamapReport.ts b/clients/admin-ui/src/types/api/models/DatamapReport.ts index 5283343ba5..b3f8cffc13 100644 --- a/clients/admin-ui/src/types/api/models/DatamapReport.ts +++ b/clients/admin-ui/src/types/api/models/DatamapReport.ts @@ -33,7 +33,6 @@ export type DatamapReport = { legal_basis_for_transfers?: Array | null; legal_name?: string | null; legitimate_interest_disclosure_url?: string | null; - link_to_processor_contract?: string | null; privacy_policy?: string | null; processes_personal_data: boolean; reason_for_exemption?: string | null; @@ -42,9 +41,7 @@ export type DatamapReport = { retention_period?: string | null; shared_categories?: Array | null; special_category_legal_basis?: string | null; - system_dependencies?: string | null; system_name: string; - third_country_safeguards?: string | null; third_parties?: string | null; uses_cookies: boolean; uses_non_cookie_access: boolean; diff --git a/clients/fidesui/src/components/types/api/models/System.ts b/clients/fidesui/src/components/types/api/models/System.ts index b17d8a75b8..ebc3a86179 100644 --- a/clients/fidesui/src/components/types/api/models/System.ts +++ b/clients/fidesui/src/components/types/api/models/System.ts @@ -79,10 +79,6 @@ export type System = { * */ privacy_declarations: Array; - /** - * A list of fides keys to model dependencies. - */ - system_dependencies?: Array; /** * * The contact details information model. diff --git a/clients/privacy-center/types/api/models/DatamapReport.ts b/clients/privacy-center/types/api/models/DatamapReport.ts index 5283343ba5..b3f8cffc13 100644 --- a/clients/privacy-center/types/api/models/DatamapReport.ts +++ b/clients/privacy-center/types/api/models/DatamapReport.ts @@ -33,7 +33,6 @@ export type DatamapReport = { legal_basis_for_transfers?: Array | null; legal_name?: string | null; legitimate_interest_disclosure_url?: string | null; - link_to_processor_contract?: string | null; privacy_policy?: string | null; processes_personal_data: boolean; reason_for_exemption?: string | null; @@ -42,9 +41,7 @@ export type DatamapReport = { retention_period?: string | null; shared_categories?: Array | null; special_category_legal_basis?: string | null; - system_dependencies?: string | null; system_name: string; - third_country_safeguards?: string | null; third_parties?: string | null; uses_cookies: boolean; uses_non_cookie_access: boolean; From 5557f79305b7be60ad6294b8960a011dc97e9e12 Mon Sep 17 00:00:00 2001 From: Lucano Vera Date: Mon, 13 Jan 2025 11:27:10 -0300 Subject: [PATCH 06/50] Fix cypress e2e pipeline (#5659) --- .github/workflows/cypress_e2e.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cypress_e2e.yml b/.github/workflows/cypress_e2e.yml index 952e4fc46d..7da0f97836 100644 --- a/.github/workflows/cypress_e2e.yml +++ b/.github/workflows/cypress_e2e.yml @@ -26,6 +26,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install Nox run: pip install nox>=2022 From 73188c53f9479cd6dbd4f3637a4fc03faec9d3cf Mon Sep 17 00:00:00 2001 From: Lucano Vera Date: Mon, 13 Jan 2025 12:06:42 -0300 Subject: [PATCH 07/50] LA-203 Update-CMP-Privacy-Center-powered-by-URL-to-https-ethyca.com (#5656) --- CHANGELOG.md | 3 +++ clients/fides-js/src/components/BrandLink.tsx | 2 +- clients/fides-js/src/components/EthycaLogo.tsx | 2 ++ clients/privacy-center/components/BrandLink.tsx | 10 +++++++++- .../privacy-center/cypress/e2e/consent-banner.cy.ts | 2 +- .../public/fides-js-components-demo.html | 1 + 6 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1862215a98..074f760587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) +### Changed +- Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) + ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) - Fixed column ordering issue in the Data Map report [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/fides-js/src/components/BrandLink.tsx b/clients/fides-js/src/components/BrandLink.tsx index dd84bb79ef..a22a207743 100644 --- a/clients/fides-js/src/components/BrandLink.tsx +++ b/clients/fides-js/src/components/BrandLink.tsx @@ -5,7 +5,7 @@ import EthycaLogo from "./EthycaLogo"; const BrandLink = () => (
( height="20" fill="currentColor" className="ethyca-logo" + role="img" + aria-label="Ethyca" > diff --git a/clients/privacy-center/components/BrandLink.tsx b/clients/privacy-center/components/BrandLink.tsx index 15976ceb06..d614c0afd6 100644 --- a/clients/privacy-center/components/BrandLink.tsx +++ b/clients/privacy-center/components/BrandLink.tsx @@ -22,9 +22,17 @@ const BrandLink = ({ right={right} textDecoration="none" _hover={{ textDecoration: "none" }} + href="https://ethyca.com/" {...props} > - Powered by + Powered by{" "} + ); }; diff --git a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts index eeb1bf582d..ce2eae5e1e 100644 --- a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts @@ -333,7 +333,7 @@ describe("Consent overlay", () => { cy.get("a.fides-brand-link").should( "have.attr", "href", - "https://fid.es/powered", + "https://ethyca.com/", ); }); }); diff --git a/clients/privacy-center/public/fides-js-components-demo.html b/clients/privacy-center/public/fides-js-components-demo.html index b60af186d4..57ce779cd7 100644 --- a/clients/privacy-center/public/fides-js-components-demo.html +++ b/clients/privacy-center/public/fides-js-components-demo.html @@ -207,6 +207,7 @@ privacyCenterUrl: "http://localhost:3001", fidesApiUrl: "http://localhost:8080/api/v1", fidesPrimaryColor: "#008000", + showFidesBrandLink: true, }, }; if (init !== "false") { From 7ea4ad58d49dfcd525344fdf1de86078797febf5 Mon Sep 17 00:00:00 2001 From: jpople Date: Mon, 13 Jan 2025 10:05:18 -0600 Subject: [PATCH 08/50] Render 'Reclassify' button in overflow menu when applicable (#5655) --- CHANGELOG.md | 1 + .../cypress/e2e/discovery-detection.cy.ts | 3 +- .../hooks/useDiscoveryResultColumns.tsx | 10 +- .../tables/cells/DiscoveryItemActionsCell.tsx | 124 ++++++++++++------ 4 files changed, 98 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074f760587..76114c602c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Changed - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) +- Changed "Reclassify" D&D button to show in an overflow menu when row actions are overcrowded [#5655](https://github.com/ethyca/fides/pull/5655) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts index 08931756b8..b6831c64dd 100644 --- a/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts +++ b/clients/admin-ui/cypress/e2e/discovery-detection.cy.ts @@ -327,7 +327,8 @@ describe("discovery and detection", () => { cy.getByTestId( "row-my_bigquery_monitor.prj-bigquery-418515.test_dataset_1", ).within(() => { - cy.getByTestId("action-Reclassify").click(); + cy.getByTestId("actions-overflow-btn").click(); + cy.getByTestId("action-reclassify").click({ force: true }); cy.wait("@confirmResource"); }); }); diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx index 6944f66bcd..c37b67bc72 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/hooks/useDiscoveryResultColumns.tsx @@ -74,7 +74,9 @@ const useDiscoveryResultColumns = ({ ), header: "Actions", - size: 180, + meta: { + width: "auto", + }, }), ]; return { columns }; @@ -137,6 +139,9 @@ const useDiscoveryResultColumns = ({ ), header: "Actions", + meta: { + width: "auto", + }, }), ]; return { columns }; @@ -194,6 +199,9 @@ const useDiscoveryResultColumns = ({ ), header: "Actions", + meta: { + width: "auto", + }, }), ]; return { columns }; diff --git a/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx b/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx index af87194d63..7d66da1103 100644 --- a/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx +++ b/clients/admin-ui/src/features/data-discovery-and-detection/tables/cells/DiscoveryItemActionsCell.tsx @@ -1,4 +1,16 @@ -import { CheckIcon, HStack, RepeatIcon, ViewOffIcon } from "fidesui"; +import { + AntButton as Button, + CheckIcon, + HStack, + Menu, + MenuButton, + MenuItem, + MenuList, + MoreIcon, + RepeatIcon, + Spacer, + ViewOffIcon, +} from "fidesui"; import { useAlert } from "~/features/common/hooks"; import { DiscoveryMonitorItem } from "~/features/data-discovery-and-detection/types/DiscoveryMonitorItem"; @@ -54,21 +66,50 @@ const DiscoveryItemActionsCell = ({ resource }: DiscoveryItemActionsProps) => { const showMuteAction = itemHasClassificationChanges || childItemsHaveClassificationChanges; + // if promote and mute are both shown, show "Reclassify" in an overflow menu + // to avoid having too many buttons in the cell + const showReclassifyInOverflow = showPromoteAction && showMuteAction; + + const handlePromote = async () => { + await promoteResourceMutation({ + staged_resource_urn: resource.urn, + }); + successAlert( + `These changes have been added to a Fides dataset. To view, navigate to "Manage datasets".`, + `Table changes confirmed`, + ); + }; + + const handleMute = async () => { + await muteResourceMutation({ + staged_resource_urn: resource.urn, + }); + successAlert( + `Ignored changes will not be added to a Fides dataset.`, + `${resource.name || "Changes"} ignored`, + ); + }; + + const handleReclassify = async () => { + await confirmResourceMutation({ + staged_resource_urn: resource.urn, + monitor_config_id: resource.monitor_config_id!, + start_classification: true, + classify_monitored_resources: true, + }); + successAlert( + `Reclassification of ${getResourceName(resource) || "the resource"} has begun. The results may take some time to appear in the “Data discovery“ tab.`, + `Reclassification started`, + ); + }; + return ( - e.stopPropagation()}> + e.stopPropagation()} gap={2}> {showPromoteAction && ( } - onClick={async () => { - await promoteResourceMutation({ - staged_resource_urn: resource.urn, - }); - successAlert( - `These changes have been added to a Fides dataset. To view, navigate to "Manage datasets".`, - `Table changes confirmed`, - ); - }} + onClick={handlePromote} disabled={anyActionIsLoading} loading={promoteIsLoading} /> @@ -77,37 +118,44 @@ const DiscoveryItemActionsCell = ({ resource }: DiscoveryItemActionsProps) => { } - onClick={async () => { - await muteResourceMutation({ - staged_resource_urn: resource.urn, - }); - successAlert( - `Ignored changes will not be added to a Fides dataset.`, - `${resource.name || "Changes"} ignored`, - ); - }} + onClick={handleMute} disabled={anyActionIsLoading} loading={muteIsLoading} /> )} - } - onClick={async () => { - await confirmResourceMutation({ - staged_resource_urn: resource.urn, - monitor_config_id: resource.monitor_config_id!, - start_classification: true, - classify_monitored_resources: true, - }); - successAlert( - `Reclassification of ${getResourceName(resource) || "the resource"} has begun. The results may take some time to appear in the “Data discovery“ tab.`, - `Reclassification started`, - ); - }} - disabled={anyActionIsLoading} - loading={confirmIsLoading} - /> + {!showReclassifyInOverflow && ( + } + onClick={handleReclassify} + disabled={anyActionIsLoading} + loading={confirmIsLoading} + /> + )} + + {showReclassifyInOverflow && ( + + } + className="w-6 gap-0" + data-testid="actions-overflow-btn" + /> + + } + data-testid="action-reclassify" + > + Reclassify + + + + )} ); }; From 83cb612e4bc7f1fe424cab8a0a54c024331dfe53 Mon Sep 17 00:00:00 2001 From: Dave Quinlan <83430497+daveqnet@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:08:18 +0000 Subject: [PATCH 09/50] chore: remove patch version pin from redis image tag (#5660) --- docker-compose.child-env.yml | 2 +- docker-compose.yml | 2 +- docker/docker-compose.minimal-config.yml | 2 +- src/fides/data/sample_project/docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.child-env.yml b/docker-compose.child-env.yml index d56871f799..108524ac6e 100644 --- a/docker-compose.child-env.yml +++ b/docker-compose.child-env.yml @@ -64,7 +64,7 @@ services: - node.labels.fides.app-db-data == true redis-child: - image: "redis:6.2.5-alpine" + image: "redis:6.2-alpine" command: redis-server --requirepass redispassword expose: - 6379 diff --git a/docker-compose.yml b/docker-compose.yml index 6ba4ae2da4..2917b0fa11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,7 +136,7 @@ services: - /fides/src/fides.egg-info redis: - image: "redis:6.2.5-alpine" + image: "redis:6.2-alpine" # AUTH option #1: no authentication at all # command: redis-server # AUTH option #2: require password diff --git a/docker/docker-compose.minimal-config.yml b/docker/docker-compose.minimal-config.yml index 15e49ea9e6..b06c1e2a7c 100644 --- a/docker/docker-compose.minimal-config.yml +++ b/docker/docker-compose.minimal-config.yml @@ -52,7 +52,7 @@ services: - node.labels.fides.app-db-data == true redis: - image: "redis:6.2.5-alpine" + image: "redis:6.2-alpine" command: redis-server --requirepass redispassword expose: - 6379 diff --git a/src/fides/data/sample_project/docker-compose.yml b/src/fides/data/sample_project/docker-compose.yml index 66de719665..bcbfd5239b 100644 --- a/src/fides/data/sample_project/docker-compose.yml +++ b/src/fides/data/sample_project/docker-compose.yml @@ -103,7 +103,7 @@ services: redis: container_name: fides-redis - image: redis:6.2.5-alpine + image: redis:6.2-alpine command: redis-server --requirepass redispassword ports: - "7379:6379" From de209feb31ce2619c9db608d41fa5d58877589cd Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 13 Jan 2025 10:02:13 -0800 Subject: [PATCH 10/50] Splitting up query config files and tests (#5591) --- CHANGELOG.md | 1 + .../bigquery_enterprise_test_dataset.yml | 552 +++-------- .../dataset/bigquery_example_test_dataset.yml | 18 - .../api/service/connectors/base_connector.py | 11 + .../service/connectors/bigquery_connector.py | 5 + .../service/connectors/postgres_connector.py | 5 + .../query_configs/bigquery_query_config.py | 28 +- .../connectors/query_configs/query_config.py | 69 +- .../query_configs/snowflake_query_config.py | 9 +- .../service/connectors/scylla_connector.py | 5 + .../service/connectors/scylla_query_config.py | 26 +- src/fides/api/task/graph_task.py | 13 +- ...s_example_custom_request_field_dataset.yml | 2 +- tests/fixtures/application_fixtures.py | 53 ++ tests/fixtures/postgres_fixtures.py | 30 + .../integration_tests/test_mariadb_task.py | 101 ++ .../ops/integration_tests/test_mssql_task.py | 101 ++ .../ops/integration_tests/test_mysql_task.py | 101 ++ .../integration_tests/test_scylladb_task.py | 190 ++++ tests/ops/integration_tests/test_sql_task.py | 885 ++---------------- .../integration_tests/test_timescale_task.py | 294 ++++++ .../connectors/test_bigquery_connector.py | 4 +- .../connectors/test_bigquery_queryconfig.py | 4 +- .../connectors/test_dynamodb_query_config.py | 129 +++ .../connectors/test_mongo_query_config.py | 283 ++++++ .../service/connectors/test_query_config.py | 576 ++++-------- .../connectors/test_scylladb_query_config.py | 47 + .../connectors/test_snowflake_query_config.py | 4 +- .../example_datasets/multiple_identities.yml | 2 - ...le_identities_with_external_dependency.yml | 2 - .../example_datasets/no_identities.yml | 2 - .../example_datasets/single_identity.yml | 2 - ...ngle_identity_with_internal_dependency.yml | 2 - ...est_bigquery_enterprise_privacy_request.py | 22 +- .../test_postgres_privacy_requests.py | 30 +- tests/ops/test_helpers/dataset_utils.py | 30 +- 36 files changed, 1914 insertions(+), 1724 deletions(-) create mode 100644 tests/ops/integration_tests/test_mariadb_task.py create mode 100644 tests/ops/integration_tests/test_mssql_task.py create mode 100644 tests/ops/integration_tests/test_mysql_task.py create mode 100644 tests/ops/integration_tests/test_scylladb_task.py create mode 100644 tests/ops/integration_tests/test_timescale_task.py create mode 100644 tests/ops/service/connectors/test_dynamodb_query_config.py create mode 100644 tests/ops/service/connectors/test_mongo_query_config.py create mode 100644 tests/ops/service/connectors/test_scylladb_query_config.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 76114c602c..9eb9994d6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Changed - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) - Changed "Reclassify" D&D button to show in an overflow menu when row actions are overcrowded [#5655](https://github.com/ethyca/fides/pull/5655) +- Removed primary key requirements for BigQuery and Postgres erasures [#5591](https://github.com/ethyca/fides/pull/5591) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/data/dataset/bigquery_enterprise_test_dataset.yml b/data/dataset/bigquery_enterprise_test_dataset.yml index 59d27e68a2..64668192d0 100644 --- a/data/dataset/bigquery_enterprise_test_dataset.yml +++ b/data/dataset/bigquery_enterprise_test_dataset.yml @@ -1,405 +1,149 @@ dataset: - - fides_key: enterprise_dsr_testing - organization_fides_key: default_organization - tags: null - name: Bigquery Enterprise Test Dataset - description: BigQuery dataset containing real data - meta: null - data_categories: null - fides_meta: - resource_id: enterprise_dsr_testing.prj-sandbox-55855.enterprise_dsr_testing - after: null - namespace: - dataset_id: enterprise_dsr_testing - project_id: prj-sandbox-55855 - collections: - - name: comments - description: null - data_categories: null - fields: - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - system.operations - fides_meta: - references: null - identity: null - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: post_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: score - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: text - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: user_display_name - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: user_id - description: null - data_categories: - - user.contact - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: null - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - fides_meta: null - - name: post_history - description: null - data_categories: null - fields: - - name: comment - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - system.operations - fides_meta: - references: null - identity: null - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: post_history_type_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: post_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: revision_guid - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: text - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: user_id - description: null - data_categories: - - system.operations - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: null - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - fides_meta: null - - name: stackoverflow_posts - description: null - data_categories: null - fields: - - name: accepted_answer_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: answer_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: body - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: comment_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: community_owned_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: favorite_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - system.operations - fides_meta: - references: null - identity: null - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: last_activity_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: last_edit_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: last_editor_display_name - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: last_editor_user_id - description: null - data_categories: - - system.operations - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: null - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: owner_display_name - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: owner_user_id - description: null - data_categories: - - system.operations - fides_meta: - references: - - dataset: enterprise_dsr_testing - field: users.id - direction: from - identity: null - primary_key: null - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: parent_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: post_type_id - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: score - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: tags - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: title - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: view_count - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - fides_meta: null - - name: users - description: null - data_categories: null - fields: - - name: about_me - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: age - description: null - data_categories: - - user - fides_meta: null - fields: null - - name: creation_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: display_name - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: down_votes - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: id - description: null - data_categories: - - user.contact - fides_meta: - references: null - identity: stackoverflow_user_id - primary_key: true - data_type: integer - length: null - return_all_elements: null - read_only: null - custom_request_field: null - fields: null - - name: last_access_date - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: location - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: profile_image_url - description: null - data_categories: - - user.contact - fides_meta: null - fields: null - - name: reputation - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: up_votes - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: views - description: null - data_categories: - - system.operations - fides_meta: null - fields: null - - name: website_url - description: null - data_categories: - - user - fides_meta: null - fields: null - fides_meta: - after: null - erase_after: - - enterprise_dsr_testing.comments - skip_processing: false - masking_strategy_override: null - partitioning: null + - fides_key: enterprise_dsr_testing + organization_fides_key: default_organization + name: Bigquery Enterprise Test Dataset + description: BigQuery dataset containing real data + fides_meta: + resource_id: enterprise_dsr_testing.prj-sandbox-55855.enterprise_dsr_testing + namespace: + dataset_id: enterprise_dsr_testing + project_id: prj-sandbox-55855 + collections: + - name: comments + fields: + - name: creation_date + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + fides_meta: + data_type: integer + - name: post_id + data_categories: [system.operations] + - name: score + data_categories: [system.operations] + - name: text + data_categories: [user.contact] + - name: user_display_name + data_categories: [user.contact] + - name: user_id + data_categories: [user.contact] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: post_history + fields: + - name: comment + data_categories: [user.contact] + - name: creation_date + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + fides_meta: + data_type: integer + - name: post_history_type_id + data_categories: [system.operations] + - name: post_id + data_categories: [system.operations] + - name: revision_guid + data_categories: [system.operations] + - name: text + data_categories: [user.contact] + - name: user_id + data_categories: [system.operations] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: stackoverflow_posts + fields: + - name: accepted_answer_id + data_categories: [system.operations] + - name: answer_count + data_categories: [system.operations] + - name: body + data_categories: [user.contact] + - name: comment_count + data_categories: [system.operations] + - name: community_owned_date + data_categories: [system.operations] + - name: creation_date + data_categories: [system.operations] + - name: favorite_count + data_categories: [system.operations] + - name: id + data_categories: [system.operations] + fides_meta: + data_type: integer + - name: last_activity_date + data_categories: [system.operations] + - name: last_edit_date + data_categories: [system.operations] + - name: last_editor_display_name + data_categories: [system.operations] + - name: last_editor_user_id + data_categories: [system.operations] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: owner_display_name + data_categories: [user.contact] + - name: owner_user_id + data_categories: [system.operations] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + data_type: integer + - name: parent_id + data_categories: [system.operations] + - name: post_type_id + data_categories: [system.operations] + - name: score + data_categories: [system.operations] + - name: tags + data_categories: [system.operations] + - name: title + data_categories: [user.contact] + - name: view_count + data_categories: [system.operations] + - name: users + fields: + - name: about_me + data_categories: [user.contact] + - name: age + data_categories: [user] + - name: creation_date + data_categories: [system.operations] + - name: display_name + data_categories: [user.contact] + - name: down_votes + data_categories: [system.operations] + - name: id + data_categories: [user.contact] + fides_meta: + identity: stackoverflow_user_id + data_type: integer + - name: last_access_date + data_categories: [system.operations] + - name: location + data_categories: [user.contact] + - name: profile_image_url + data_categories: [user.contact] + - name: reputation + data_categories: [system.operations] + - name: up_votes + data_categories: [system.operations] + - name: views + data_categories: [system.operations] + - name: website_url + data_categories: [user] + fides_meta: + erase_after: + - enterprise_dsr_testing.comments + skip_processing: false diff --git a/data/dataset/bigquery_example_test_dataset.yml b/data/dataset/bigquery_example_test_dataset.yml index 11fdac1aba..c4ea16cb44 100644 --- a/data/dataset/bigquery_example_test_dataset.yml +++ b/data/dataset/bigquery_example_test_dataset.yml @@ -13,8 +13,6 @@ dataset: data_categories: [user.contact.address.street] - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: state data_categories: [user.contact.address.state] - name: street @@ -53,8 +51,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: @@ -80,8 +76,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: @@ -98,8 +92,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: time data_categories: [user.sensor] @@ -114,8 +106,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: shipping_address_id data_categories: [system.operations] fides_meta: @@ -166,8 +156,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: name data_categories: [user.financial] - name: preferred @@ -177,8 +165,6 @@ dataset: fields: - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: name data_categories: [system.operations] - name: price @@ -193,8 +179,6 @@ dataset: data_type: string - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: month data_categories: [system.operations] - name: name @@ -227,8 +211,6 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: opened data_categories: [system.operations] diff --git a/src/fides/api/service/connectors/base_connector.py b/src/fides/api/service/connectors/base_connector.py index ca3439f523..e1f735df1c 100644 --- a/src/fides/api/service/connectors/base_connector.py +++ b/src/fides/api/service/connectors/base_connector.py @@ -132,3 +132,14 @@ def execute_standalone_retrieval_query( raise NotImplementedError( "execute_standalone_retrieval_query must be implemented in a concrete subclass" ) + + @property + def requires_primary_keys(self) -> bool: + """ + Indicates if datasets linked to this connector require primary keys for erasures. + Defaults to True. + """ + + # Defaulting to true for now so we can keep the default behavior and + # incrementally determine the need for primary keys across all connectors + return True diff --git a/src/fides/api/service/connectors/bigquery_connector.py b/src/fides/api/service/connectors/bigquery_connector.py index 8b51f90842..ae6fe4b909 100644 --- a/src/fides/api/service/connectors/bigquery_connector.py +++ b/src/fides/api/service/connectors/bigquery_connector.py @@ -33,6 +33,11 @@ class BigQueryConnector(SQLConnector): secrets_schema = BigQuerySchema + @property + def requires_primary_keys(self) -> bool: + """BigQuery does not have the concept of primary keys so they're not required for erasures.""" + return False + # Overrides BaseConnector.build_uri def build_uri(self) -> str: """Build URI of format""" diff --git a/src/fides/api/service/connectors/postgres_connector.py b/src/fides/api/service/connectors/postgres_connector.py index 5354d4ec13..2abafc01c8 100644 --- a/src/fides/api/service/connectors/postgres_connector.py +++ b/src/fides/api/service/connectors/postgres_connector.py @@ -19,6 +19,11 @@ class PostgreSQLConnector(SQLConnector): secrets_schema = PostgreSQLSchema + @property + def requires_primary_keys(self) -> bool: + """Postgres allows arbitrary columns in the WHERE clause for updates so primary keys are not required.""" + return False + def build_uri(self) -> str: """Build URI of format postgresql://[user[:password]@][netloc][:port][/dbname]""" config = self.secrets_schema(**self.configuration.secrets or {}) diff --git a/src/fides/api/service/connectors/query_configs/bigquery_query_config.py b/src/fides/api/service/connectors/query_configs/bigquery_query_config.py index 681e2b9c60..6060ff5822 100644 --- a/src/fides/api/service/connectors/query_configs/bigquery_query_config.py +++ b/src/fides/api/service/connectors/query_configs/bigquery_query_config.py @@ -123,15 +123,15 @@ def generate_update( TODO: DRY up this method and `generate_delete` a bit """ update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) - non_empty_primary_keys: Dict[str, Field] = filter_nonempty_values( + non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values( { fpath.string_path: fld.cast(row[fpath.string_path]) - for fpath, fld in self.primary_key_field_paths.items() + for fpath, fld in self.reference_field_paths.items() if fpath.string_path in row } ) - valid = len(non_empty_primary_keys) > 0 and update_value_map + valid = len(non_empty_reference_field_keys) > 0 and update_value_map if not valid: logger.warning( "There is not enough data to generate a valid update statement for {}", @@ -140,8 +140,8 @@ def generate_update( return [] table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True) - pk_clauses: List[ColumnElement] = [ - getattr(table.c, k) == v for k, v in non_empty_primary_keys.items() + where_clauses: List[ColumnElement] = [ + getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items() ] if self.partitioning: @@ -153,13 +153,13 @@ def generate_update( for partition_clause in partition_clauses: partitioned_queries.append( table.update() - .where(*(pk_clauses + [text(partition_clause)])) + .where(*(where_clauses + [text(partition_clause)])) .values(**update_value_map) ) return partitioned_queries - return [table.update().where(*pk_clauses).values(**update_value_map)] + return [table.update().where(*where_clauses).values(**update_value_map)] def generate_delete(self, row: Row, client: Engine) -> List[Delete]: """Returns a List of SQLAlchemy DELETE statements for BigQuery. Does not actually execute the delete statement. @@ -172,15 +172,15 @@ def generate_delete(self, row: Row, client: Engine) -> List[Delete]: TODO: DRY up this method and `generate_update` a bit """ - non_empty_primary_keys: Dict[str, Field] = filter_nonempty_values( + non_empty_reference_field_keys: Dict[str, Field] = filter_nonempty_values( { fpath.string_path: fld.cast(row[fpath.string_path]) - for fpath, fld in self.primary_key_field_paths.items() + for fpath, fld in self.reference_field_paths.items() if fpath.string_path in row } ) - valid = len(non_empty_primary_keys) > 0 + valid = len(non_empty_reference_field_keys) > 0 if not valid: logger.warning( "There is not enough data to generate a valid DELETE statement for {}", @@ -189,8 +189,8 @@ def generate_delete(self, row: Row, client: Engine) -> List[Delete]: return [] table = Table(self._generate_table_name(), MetaData(bind=client), autoload=True) - pk_clauses: List[ColumnElement] = [ - getattr(table.c, k) == v for k, v in non_empty_primary_keys.items() + where_clauses: List[ColumnElement] = [ + getattr(table.c, k) == v for k, v in non_empty_reference_field_keys.items() ] if self.partitioning: @@ -202,9 +202,9 @@ def generate_delete(self, row: Row, client: Engine) -> List[Delete]: for partition_clause in partition_clauses: partitioned_queries.append( - table.delete().where(*(pk_clauses + [text(partition_clause)])) + table.delete().where(*(where_clauses + [text(partition_clause)])) ) return partitioned_queries - return [table.delete().where(*pk_clauses)] + return [table.delete().where(*where_clauses)] diff --git a/src/fides/api/service/connectors/query_configs/query_config.py b/src/fides/api/service/connectors/query_configs/query_config.py index 6e868964af..9f5ddb0251 100644 --- a/src/fides/api/service/connectors/query_configs/query_config.py +++ b/src/fides/api/service/connectors/query_configs/query_config.py @@ -100,6 +100,15 @@ def primary_key_field_paths(self) -> Dict[FieldPath, Field]: if field.primary_key } + @property + def reference_field_paths(self) -> Dict[FieldPath, Field]: + """Mapping of FieldPaths to Fields that have incoming identity or dataset references""" + return { + field_path: field + for field_path, field in self.field_map().items() + if field_path in {edge.f2.field_path for edge in self.node.incoming_edges} + } + def query_sources(self) -> Dict[str, List[CollectionAddress]]: """Display the input collection(s) for each query key for display purposes. @@ -412,14 +421,16 @@ def generate_query_without_tuples( # pylint: disable=R0914 def get_update_stmt( self, update_clauses: List[str], - pk_clauses: List[str], + where_clauses: List[str], ) -> str: """Returns a SQL UPDATE statement to fit SQL syntax.""" - return f"UPDATE {self.node.address.collection} SET {', '.join(update_clauses)} WHERE {' AND '.join(pk_clauses)}" + return f"UPDATE {self.node.address.collection} SET {', '.join(update_clauses)} WHERE {' AND '.join(where_clauses)}" @abstractmethod def get_update_clauses( - self, update_value_map: Dict[str, Any], non_empty_primary_keys: Dict[str, Field] + self, + update_value_map: Dict[str, Any], + where_clause_fields: Dict[str, Field], ) -> List[str]: """Returns a list of update clauses for the update statement.""" @@ -428,7 +439,7 @@ def format_query_stmt(self, query_str: str, update_value_map: Dict[str, Any]) -> """Returns a formatted update statement in the appropriate dialect.""" @abstractmethod - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" def generate_update_stmt( @@ -436,7 +447,8 @@ def generate_update_stmt( ) -> Optional[T]: """Returns an update statement in generic SQL-ish dialect.""" update_value_map: Dict[str, Any] = self.update_value_map(row, policy, request) - non_empty_primary_keys: Dict[str, Field] = filter_nonempty_values( + + non_empty_primary_key_fields: Dict[str, Field] = filter_nonempty_values( { fpath.string_path: fld.cast(row[fpath.string_path]) for fpath, fld in self.primary_key_field_paths.items() @@ -444,17 +456,30 @@ def generate_update_stmt( } ) + non_empty_reference_fields: Dict[str, Field] = filter_nonempty_values( + { + fpath.string_path: fld.cast(row[fpath.string_path]) + for fpath, fld in self.reference_field_paths.items() + if fpath.string_path in row + } + ) + + # Create parameter mappings with masked_ prefix for SET values + param_map = { + **{f"masked_{k}": v for k, v in update_value_map.items()}, + **non_empty_primary_key_fields, + **non_empty_reference_fields, + } + update_clauses = self.get_update_clauses( - update_value_map, non_empty_primary_keys + {k: f"masked_{k}" for k in update_value_map}, + non_empty_primary_key_fields or non_empty_reference_fields, ) - pk_clauses = self.format_key_map_for_update_stmt( - list(non_empty_primary_keys.keys()) + where_clauses = self.format_key_map_for_update_stmt( + {k: k for k in non_empty_primary_key_fields or non_empty_reference_fields} ) - for k, v in non_empty_primary_keys.items(): - update_value_map[k] = v - - valid = len(pk_clauses) > 0 and len(update_clauses) > 0 + valid = len(where_clauses) > 0 and len(update_clauses) > 0 if not valid: logger.warning( "There is not enough data to generate a valid update statement for {}", @@ -462,12 +487,9 @@ def generate_update_stmt( ) return None - query_str = self.get_update_stmt( - update_clauses, - pk_clauses, - ) - logger.info("query = {}, params = {}", Pii(query_str), Pii(update_value_map)) - return self.format_query_stmt(query_str, update_value_map) + query_str = self.get_update_stmt(update_clauses, where_clauses) + logger.info("query = {}, params = {}", Pii(query_str), Pii(param_map)) + return self.format_query_stmt(query_str, param_map) class SQLQueryConfig(SQLLikeQueryConfig[Executable]): @@ -538,16 +560,17 @@ def generate_query( ) return None - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" - fields.sort() - return [f"{k} = :{k}" for k in fields] + return [f"{k} = :{v}" for k, v in sorted(param_map.items())] def get_update_clauses( - self, update_value_map: Dict[str, Any], non_empty_primary_keys: Dict[str, Field] + self, + update_value_map: Dict[str, Any], + where_clause_fields: Dict[str, Field], ) -> List[str]: """Returns a list of update clauses for the update statement.""" - return self.format_key_map_for_update_stmt(list(update_value_map.keys())) + return self.format_key_map_for_update_stmt(update_value_map) def format_query_stmt( self, query_str: str, update_value_map: Dict[str, Any] diff --git a/src/fides/api/service/connectors/query_configs/snowflake_query_config.py b/src/fides/api/service/connectors/query_configs/snowflake_query_config.py index 574e1ea1b1..ec640191d8 100644 --- a/src/fides/api/service/connectors/query_configs/snowflake_query_config.py +++ b/src/fides/api/service/connectors/query_configs/snowflake_query_config.py @@ -59,15 +59,14 @@ def get_formatted_query_string( """Returns a query string with double quotation mark formatting as required by Snowflake syntax.""" return f'SELECT {field_list} FROM {self._generate_table_name()} WHERE ({" OR ".join(clauses)})' - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" - fields.sort() - return [f'"{k}" = :{k}' for k in fields] + return [f'"{k}" = :{v}' for k, v in sorted(param_map.items())] def get_update_stmt( self, update_clauses: List[str], - pk_clauses: List[str], + where_clauses: List[str], ) -> str: """Returns a parameterized update statement in Snowflake dialect.""" - return f'UPDATE {self._generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(pk_clauses)}' + return f'UPDATE {self._generate_table_name()} SET {", ".join(update_clauses)} WHERE {" AND ".join(where_clauses)}' diff --git a/src/fides/api/service/connectors/scylla_connector.py b/src/fides/api/service/connectors/scylla_connector.py index 43a821930c..ff17674b88 100644 --- a/src/fides/api/service/connectors/scylla_connector.py +++ b/src/fides/api/service/connectors/scylla_connector.py @@ -28,6 +28,11 @@ class ScyllaConnectorMissingKeyspace(Exception): class ScyllaConnector(BaseConnector[Cluster]): """Scylla Connector""" + @property + def requires_primary_keys(self) -> bool: + """ScyllaDB requires primary keys for erasures.""" + return True + def build_uri(self) -> str: """ Builds URI - Not yet implemented diff --git a/src/fides/api/service/connectors/scylla_query_config.py b/src/fides/api/service/connectors/scylla_query_config.py index 2a72270a40..1fa52d573d 100644 --- a/src/fides/api/service/connectors/scylla_query_config.py +++ b/src/fides/api/service/connectors/scylla_query_config.py @@ -70,21 +70,27 @@ def generate_query( ) -> Optional[ScyllaDBStatement]: return self.generate_query_without_tuples(input_data, policy) - def format_key_map_for_update_stmt(self, fields: List[str]) -> List[str]: + def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]: """Adds the appropriate formatting for update statements in this datastore.""" - fields.sort() - return [f"{k} = %({k})s" for k in fields] + return [f"{k} = %({v})s" for k, v in sorted(param_map.items())] def get_update_clauses( - self, update_value_map: Dict[str, Any], non_empty_primary_keys: Dict[str, Field] + self, + update_value_map: Dict[str, Any], + where_clause_fields: Dict[str, Field], ) -> List[str]: - """Returns a list of update clauses for the update statement.""" + """Returns a list of update clauses for the update statement. + + Omits primary key fields from updates since ScyllaDB prohibits + updating primary key fields. + """ + return self.format_key_map_for_update_stmt( - [ - key - for key in update_value_map.keys() - if key not in non_empty_primary_keys - ] + { + key: value + for key, value in update_value_map.items() + if key not in where_clause_fields + } ) def format_query_data_name(self, query_data_name: str) -> str: diff --git a/src/fides/api/task/graph_task.py b/src/fides/api/task/graph_task.py index 6b78b57297..145094ea25 100644 --- a/src/fides/api/task/graph_task.py +++ b/src/fides/api/task/graph_task.py @@ -603,12 +603,19 @@ def erasure_request( *erasure_prereqs: int, # TODO Remove when we stop support for DSR 2.0. DSR 3.0 enforces with downstream_tasks. ) -> int: """Run erasure request""" + # if there is no primary key specified in the graph node configuration # note this in the execution log and perform no erasures on this node - if not self.execution_node.collection.contains_field(lambda f: f.primary_key): + if ( + self.connector.requires_primary_keys + and not self.execution_node.collection.contains_field( + lambda f: f.primary_key + ) + ): logger.warning( - "No erasures on {} as there is no primary_key defined.", + 'Skipping erasures on "{}" as the "{}" connector requires a primary key to be defined in one of the collection fields, but none was found.', self.execution_node.address, + self.connector.configuration.connection_type, ) if self.request_task.id: # For DSR 3.0, largely for testing. DSR 3.0 uses Request Task status @@ -617,7 +624,7 @@ def erasure_request( # TODO Remove when we stop support for DSR 2.0 self.resources.cache_erasure(self.key.value, 0) self.update_status( - "No values were erased since no primary key was defined for this collection", + "No values were erased since no primary key was defined in any of the fields for this collection", None, ActionType.erasure, ExecutionLogStatus.complete, diff --git a/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml b/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml index 96b58645d4..0171258b1d 100644 --- a/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml +++ b/src/fides/data/sample_project/sample_resources/postgres_example_custom_request_field_dataset.yml @@ -2,7 +2,7 @@ dataset: - fides_key: postgres_example_custom_request_field_dataset data_categories: [] description: Postgres example dataset with a custom request field - name: Postgrex Example Custom Request Field Dataset + name: Postgres Example Custom Request Field Dataset collections: - name: dynamic_email_address_config fields: diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 8356c42111..d030919aed 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -864,6 +864,59 @@ def erasure_policy( "rule_id": erasure_rule.id, }, ) + + yield erasure_policy + try: + rule_target.delete(db) + except ObjectDeletedError: + pass + try: + erasure_rule.delete(db) + except ObjectDeletedError: + pass + try: + erasure_policy.delete(db) + except ObjectDeletedError: + pass + + +@pytest.fixture(scope="function") +def erasure_policy_address_city( + db: Session, + oauth_client: ClientDetail, +) -> Generator: + erasure_policy = Policy.create( + db=db, + data={ + "name": "example erasure policy", + "key": "example_erasure_policy", + "client_id": oauth_client.id, + }, + ) + + erasure_rule = Rule.create( + db=db, + data={ + "action_type": ActionType.erasure.value, + "client_id": oauth_client.id, + "name": "Erasure Rule", + "policy_id": erasure_policy.id, + "masking_strategy": { + "strategy": "null_rewrite", + "configuration": {}, + }, + }, + ) + + rule_target = RuleTarget.create( + db=db, + data={ + "client_id": oauth_client.id, + "data_category": DataCategory("user.contact.address.city").value, + "rule_id": erasure_rule.id, + }, + ) + yield erasure_policy try: rule_target.delete(db) diff --git a/tests/fixtures/postgres_fixtures.py b/tests/fixtures/postgres_fixtures.py index 5e2aaec047..6db641fd1d 100644 --- a/tests/fixtures/postgres_fixtures.py +++ b/tests/fixtures/postgres_fixtures.py @@ -2,6 +2,7 @@ from uuid import uuid4 import pytest +from fideslang.models import Dataset as FideslangDataset from sqlalchemy.orm import Session from sqlalchemy.orm.exc import ObjectDeletedError from sqlalchemy_utils.functions import drop_database @@ -23,6 +24,7 @@ from fides.api.models.sql_models import System from fides.api.service.connectors import PostgreSQLConnector from fides.config import CONFIG +from tests.ops.test_helpers.dataset_utils import remove_primary_keys from tests.ops.test_helpers.db_utils import seed_postgres_data from .application_fixtures import integration_secrets @@ -111,6 +113,34 @@ def postgres_example_test_dataset_config_read_access( ctl_dataset.delete(db=db) +@pytest.fixture +def postgres_example_test_dataset_config_read_access_without_primary_keys( + read_connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], +) -> Generator: + postgres_dataset = example_datasets[0] + fides_key = postgres_dataset["fides_key"] + + dataset = FideslangDataset(**postgres_dataset) + updated_dataset = remove_primary_keys(dataset) + ctl_dataset = CtlDataset.create_from_dataset_dict( + db, updated_dataset.model_dump(mode="json") + ) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": read_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + @pytest.fixture def postgres_example_test_dataset_config_skipped_login_collection( read_connection_config: ConnectionConfig, diff --git a/tests/ops/integration_tests/test_mariadb_task.py b/tests/ops/integration_tests/test_mariadb_task.py new file mode 100644 index 0000000000..3951a2830a --- /dev/null +++ b/tests/ops/integration_tests/test_mariadb_task.py @@ -0,0 +1,101 @@ +import pytest + +from fides.api.models.privacy_request import ExecutionLog + +from ...conftest import access_runner_tester +from ..graph.graph_test_util import assert_rows_match, records_matching_fields +from ..task.traversal_data import integration_db_graph + + +@pytest.mark.integration_mariadb +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_mariadb_access_request_task( + db, + policy, + connection_config_mariadb, + mariadb_integration_db, + dsr_version, + request, + privacy_request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph("my_maria_db_1"), + [connection_config_mariadb], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v["my_maria_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_maria_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_maria_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_maria_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_maria_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_maria_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_maria_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_maria_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_maria_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) diff --git a/tests/ops/integration_tests/test_mssql_task.py b/tests/ops/integration_tests/test_mssql_task.py new file mode 100644 index 0000000000..6bc23eeda0 --- /dev/null +++ b/tests/ops/integration_tests/test_mssql_task.py @@ -0,0 +1,101 @@ +import pytest + +from fides.api.models.privacy_request import ExecutionLog + +from ...conftest import access_runner_tester +from ..graph.graph_test_util import assert_rows_match, records_matching_fields +from ..task.traversal_data import integration_db_graph + + +@pytest.mark.integration_mssql +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_mssql_access_request_task( + db, + policy, + connection_config_mssql, + mssql_integration_db, + privacy_request, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph("my_mssql_db_1"), + [connection_config_mssql], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v["my_mssql_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_mssql_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_mssql_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_mssql_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_mssql_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mssql_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_mssql_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) diff --git a/tests/ops/integration_tests/test_mysql_task.py b/tests/ops/integration_tests/test_mysql_task.py new file mode 100644 index 0000000000..40551dd4d9 --- /dev/null +++ b/tests/ops/integration_tests/test_mysql_task.py @@ -0,0 +1,101 @@ +import pytest + +from fides.api.models.privacy_request import ExecutionLog + +from ...conftest import access_runner_tester +from ..graph.graph_test_util import assert_rows_match, records_matching_fields +from ..task.traversal_data import integration_db_graph + + +@pytest.mark.integration +@pytest.mark.integration_mysql +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_mysql_access_request_task( + db, + policy, + connection_config_mysql, + mysql_integration_db, + privacy_request, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph("my_mysql_db_1"), + [connection_config_mysql], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v["my_mysql_db_1:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v["my_mysql_db_1:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v["my_mysql_db_1:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v["my_mysql_db_1:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v["my_mysql_db_1:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="customer" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="address" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, dataset_name="my_mysql_db_1", collection_name="orders" + ) + ) + > 0 + ) + assert ( + len( + records_matching_fields( + logs, + dataset_name="my_mysql_db_1", + collection_name="payment_card", + ) + ) + > 0 + ) diff --git a/tests/ops/integration_tests/test_scylladb_task.py b/tests/ops/integration_tests/test_scylladb_task.py new file mode 100644 index 0000000000..8ced1317ad --- /dev/null +++ b/tests/ops/integration_tests/test_scylladb_task.py @@ -0,0 +1,190 @@ +import pytest +from sqlalchemy.orm import Session + +from fides.api.models.privacy_request import ExecutionLogStatus, PrivacyRequest +from fides.api.service.connectors.scylla_connector import ScyllaConnectorMissingKeyspace +from fides.api.task.graph_task import get_cached_data_for_erasures + +from ...conftest import access_runner_tester, erasure_runner_tester +from ..graph.graph_test_util import assert_rows_match, erasure_policy +from ..task.traversal_data import integration_scylladb_graph + + +@pytest.mark.integration +@pytest.mark.integration_scylladb +@pytest.mark.asyncio +class TestScyllaDSRs: + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_2_0"], + ) + async def test_scylladb_access_request_task_no_keyspace_dsr2( + self, + db: Session, + policy, + integration_scylladb_config, + scylladb_integration_no_keyspace, + privacy_request, + dsr_version, + request, + ) -> None: + request.getfixturevalue(dsr_version) + + with pytest.raises(ScyllaConnectorMissingKeyspace) as err: + v = access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example"), + [integration_scylladb_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert ( + "No keyspace provided in the ScyllaDB configuration for connector scylla_example" + in str(err.value) + ) + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0"], + ) + async def test_scylladb_access_request_task_no_keyspace_dsr3( + self, + db, + policy, + integration_scylladb_config, + scylladb_integration_no_keyspace, + privacy_request: PrivacyRequest, + dsr_version, + request, + ) -> None: + request.getfixturevalue(dsr_version) + v = access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example"), + [integration_scylladb_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert v == {} + assert ( + privacy_request.access_tasks.count() == 6 + ) # There's 4 tables plus the root and terminal "dummy" tasks + + # Root task should be completed + assert privacy_request.access_tasks.first().collection_name == "__ROOT__" + assert ( + privacy_request.access_tasks.first().status == ExecutionLogStatus.complete + ) + + # All other tasks should be error + for access_task in privacy_request.access_tasks.offset(1): + assert access_task.status == ExecutionLogStatus.error + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_2_0", "use_dsr_3_0"], + ) + async def test_scylladb_access_request_task( + self, + db, + policy, + integration_scylladb_config_with_keyspace, + scylla_reset_db, + scylladb_integration_with_keyspace, + privacy_request, + dsr_version, + request, + ) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + results = access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example_with_keyspace"), + [integration_scylladb_config_with_keyspace], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + results["scylla_example_with_keyspace:users"], + min_size=1, + keys=[ + "age", + "alternative_contacts", + "do_not_contact", + "email", + "name", + "last_contacted", + "logins", + "states_lived", + ], + ) + assert_rows_match( + results["scylla_example_with_keyspace:user_activity"], + min_size=3, + keys=["timestamp", "user_agent", "activity_type"], + ) + assert_rows_match( + results["scylla_example_with_keyspace:payment_methods"], + min_size=2, + keys=["card_number", "expiration_date"], + ) + assert_rows_match( + results["scylla_example_with_keyspace:orders"], + min_size=2, + keys=["order_amount", "order_date", "order_description"], + ) + + @pytest.mark.parametrize( + "dsr_version", + ["use_dsr_2_0", "use_dsr_3_0"], + ) + async def test_scylladb_erasure_task( + self, + db, + integration_scylladb_config_with_keyspace, + scylladb_integration_with_keyspace, + scylla_reset_db, + privacy_request, + dsr_version, + request, + ): + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + seed_email = "customer-1@example.com" + + policy = erasure_policy( + db, "user.name", "user.behavior", "user.device", "user.payment" + ) + privacy_request.policy_id = policy.id + privacy_request.save(db) + + graph = integration_scylladb_graph("scylla_example_with_keyspace") + access_runner_tester( + privacy_request, + policy, + integration_scylladb_graph("scylla_example_with_keyspace"), + [integration_scylladb_config_with_keyspace], + {"email": seed_email}, + db, + ) + results = erasure_runner_tester( + privacy_request, + policy, + graph, + [integration_scylladb_config_with_keyspace], + {"email": seed_email}, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + assert results == { + "scylla_example_with_keyspace:user_activity": 3, + "scylla_example_with_keyspace:users": 1, + "scylla_example_with_keyspace:payment_methods": 2, + "scylla_example_with_keyspace:orders": 2, + } diff --git a/tests/ops/integration_tests/test_sql_task.py b/tests/ops/integration_tests/test_sql_task.py index 298d77229a..bbac30df82 100644 --- a/tests/ops/integration_tests/test_sql_task.py +++ b/tests/ops/integration_tests/test_sql_task.py @@ -6,30 +6,16 @@ import pytest from fideslang import Dataset from sqlalchemy import text -from sqlalchemy.orm import Session - -from fides.api.graph.config import ( - Collection, - CollectionAddress, - FieldAddress, - GraphDataset, - ScalarField, -) + +from fides.api.graph.config import Collection, FieldAddress, GraphDataset, ScalarField from fides.api.graph.data_type import DataType, StringTypeConverter from fides.api.graph.graph import DatasetGraph, Edge, Node from fides.api.graph.traversal import TraversalNode from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import convert_dataset_to_graph from fides.api.models.policy import ActionType, Policy, Rule, RuleTarget -from fides.api.models.privacy_request import ( - ExecutionLog, - ExecutionLogStatus, - PrivacyRequest, - PrivacyRequestStatus, - RequestTask, -) +from fides.api.models.privacy_request import ExecutionLog, RequestTask from fides.api.service.connectors import get_connector -from fides.api.service.connectors.scylla_connector import ScyllaConnectorMissingKeyspace from fides.api.task.filter_results import filter_data_categories from fides.api.task.graph_task import get_cached_data_for_erasures from fides.config import CONFIG @@ -42,12 +28,7 @@ field, records_matching_fields, ) -from ..task.traversal_data import ( - integration_db_graph, - integration_scylladb_graph, - postgres_db_graph_dataset, - str_converter, -) +from ..task.traversal_data import integration_db_graph, postgres_db_graph_dataset @pytest.mark.integration_postgres @@ -57,7 +38,7 @@ "dsr_version", ["use_dsr_3_0", "use_dsr_2_0"], ) -async def test_sql_erasure_ignores_collections_without_pk( +async def test_sql_erasure_does_not_ignore_collections_without_pk( db, postgres_inserts, integration_postgres_config, @@ -116,7 +97,7 @@ async def test_sql_erasure_ignores_collections_without_pk( .all() ) logs = [log.__dict__ for log in logs] - # since address has no primary_key=True field, it's erasure is skipped + # erasure is not skipped since primary_key is not required assert ( len( records_matching_fields( @@ -126,13 +107,13 @@ async def test_sql_erasure_ignores_collections_without_pk( message="No values were erased since no primary key was defined for this collection", ) ) - == 1 + == 0 ) assert v == { "postgres_example:customer": 1, "postgres_example:payment_card": 0, "postgres_example:orders": 0, - "postgres_example:address": 0, + "postgres_example:address": 2, } @@ -456,516 +437,55 @@ async def test_postgres_privacy_requests_against_non_default_schema( db, ) - # Confirm data retrieved from backup_schema, not public schema. This data only exists in the backup_schema. - assert access_results == { - f"{database_name}:address": [ - { - "id": 7, - "street": "Test Street", - "city": "Test Town", - "state": "TX", - "zip": "79843", - } - ], - f"{database_name}:payment_card": [], - f"{database_name}:orders": [], - f"{database_name}:customer": [ - { - "id": 1, - "name": "Johanna Customer", - "email": "customer-500@example.com", - "address_id": 7, - } - ], - } - - erasure_results = erasure_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [postgres_connection_config_with_schema], - {"email": customer_email}, - get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), - db, - ) - - # Confirm record masked in non-default schema - assert erasure_results == { - f"{database_name}:customer": 1, - f"{database_name}:payment_card": 0, - f"{database_name}:orders": 0, - f"{database_name}:address": 0, - }, "Only one record on customer table has targeted data category" - customer_records = postgres_integration_db.execute( - text("select * from backup_schema.customer where id = 1;") - ) - johanna_record = [c for c in customer_records][0] - assert johanna_record.email == customer_email # Not masked - assert johanna_record.name is None # Masked by erasure request - - -@pytest.mark.integration_mssql -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_mssql_access_request_task( - db, - policy, - connection_config_mssql, - mssql_integration_db, - privacy_request, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph("my_mssql_db_1"), - [connection_config_mssql], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v["my_mssql_db_1:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v["my_mssql_db_1:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v["my_mssql_db_1:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v["my_mssql_db_1:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v["my_mssql_db_1:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mssql_db_1", collection_name="customer" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mssql_db_1", collection_name="address" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mssql_db_1", collection_name="orders" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name="my_mssql_db_1", - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration -@pytest.mark.integration_mysql -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_mysql_access_request_task( - db, - policy, - connection_config_mysql, - mysql_integration_db, - privacy_request, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph("my_mysql_db_1"), - [connection_config_mysql], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v["my_mysql_db_1:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v["my_mysql_db_1:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v["my_mysql_db_1:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v["my_mysql_db_1:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v["my_mysql_db_1:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mysql_db_1", collection_name="customer" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mysql_db_1", collection_name="address" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_mysql_db_1", collection_name="orders" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name="my_mysql_db_1", - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration_mariadb -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_mariadb_access_request_task( - db, - policy, - connection_config_mariadb, - mariadb_integration_db, - dsr_version, - request, - privacy_request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph("my_maria_db_1"), - [connection_config_mariadb], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v["my_maria_db_1:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v["my_maria_db_1:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v["my_maria_db_1:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v["my_maria_db_1:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v["my_maria_db_1:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - assert ( - len( - records_matching_fields( - logs, dataset_name="my_maria_db_1", collection_name="customer" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_maria_db_1", collection_name="address" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, dataset_name="my_maria_db_1", collection_name="orders" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name="my_maria_db_1", - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration -@pytest.mark.integration_scylladb -@pytest.mark.asyncio -class TestScyllaDSRs: - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_2_0"], - ) - async def test_scylladb_access_request_task_no_keyspace_dsr2( - self, - db: Session, - policy, - integration_scylladb_config, - scylladb_integration_no_keyspace, - privacy_request, - dsr_version, - request, - ) -> None: - request.getfixturevalue(dsr_version) - - with pytest.raises(ScyllaConnectorMissingKeyspace) as err: - v = access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example"), - [integration_scylladb_config], - {"email": "customer-1@example.com"}, - db, - ) - - assert ( - "No keyspace provided in the ScyllaDB configuration for connector scylla_example" - in str(err.value) - ) - - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0"], - ) - async def test_scylladb_access_request_task_no_keyspace_dsr3( - self, - db, - policy, - integration_scylladb_config, - scylladb_integration_no_keyspace, - privacy_request: PrivacyRequest, - dsr_version, - request, - ) -> None: - request.getfixturevalue(dsr_version) - v = access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example"), - [integration_scylladb_config], - {"email": "customer-1@example.com"}, - db, - ) - - assert v == {} - assert ( - privacy_request.access_tasks.count() == 6 - ) # There's 4 tables plus the root and terminal "dummy" tasks - - # Root task should be completed - assert privacy_request.access_tasks.first().collection_name == "__ROOT__" - assert ( - privacy_request.access_tasks.first().status == ExecutionLogStatus.complete - ) - - # All other tasks should be error - for access_task in privacy_request.access_tasks.offset(1): - assert access_task.status == ExecutionLogStatus.error - - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_2_0", "use_dsr_3_0"], - ) - async def test_scylladb_access_request_task( - self, - db, - policy, - integration_scylladb_config_with_keyspace, - scylla_reset_db, - scylladb_integration_with_keyspace, - privacy_request, - dsr_version, - request, - ) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - results = access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example_with_keyspace"), - [integration_scylladb_config_with_keyspace], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - results["scylla_example_with_keyspace:users"], - min_size=1, - keys=[ - "age", - "alternative_contacts", - "do_not_contact", - "email", - "name", - "last_contacted", - "logins", - "states_lived", - ], - ) - assert_rows_match( - results["scylla_example_with_keyspace:user_activity"], - min_size=3, - keys=["timestamp", "user_agent", "activity_type"], - ) - assert_rows_match( - results["scylla_example_with_keyspace:payment_methods"], - min_size=2, - keys=["card_number", "expiration_date"], - ) - assert_rows_match( - results["scylla_example_with_keyspace:orders"], - min_size=2, - keys=["order_amount", "order_date", "order_description"], - ) - - @pytest.mark.parametrize( - "dsr_version", - ["use_dsr_2_0", "use_dsr_3_0"], - ) - async def test_scylladb_erasure_task( - self, - db, - integration_scylladb_config_with_keyspace, - scylladb_integration_with_keyspace, - scylla_reset_db, - privacy_request, - dsr_version, - request, - ): - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - seed_email = "customer-1@example.com" + # Confirm data retrieved from backup_schema, not public schema. This data only exists in the backup_schema. + assert access_results == { + f"{database_name}:address": [ + { + "id": 7, + "street": "Test Street", + "city": "Test Town", + "state": "TX", + "zip": "79843", + } + ], + f"{database_name}:payment_card": [], + f"{database_name}:orders": [], + f"{database_name}:customer": [ + { + "id": 1, + "name": "Johanna Customer", + "email": "customer-500@example.com", + "address_id": 7, + } + ], + } - policy = erasure_policy( - db, "user.name", "user.behavior", "user.device", "user.payment" - ) - privacy_request.policy_id = policy.id - privacy_request.save(db) + erasure_results = erasure_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [postgres_connection_config_with_schema], + {"email": customer_email}, + get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), + db, + ) - graph = integration_scylladb_graph("scylla_example_with_keyspace") - access_runner_tester( - privacy_request, - policy, - integration_scylladb_graph("scylla_example_with_keyspace"), - [integration_scylladb_config_with_keyspace], - {"email": seed_email}, - db, - ) - results = erasure_runner_tester( - privacy_request, - policy, - graph, - [integration_scylladb_config_with_keyspace], - {"email": seed_email}, - get_cached_data_for_erasures(privacy_request.id), - db, - ) - assert results == { - "scylla_example_with_keyspace:user_activity": 3, - "scylla_example_with_keyspace:users": 1, - "scylla_example_with_keyspace:payment_methods": 2, - "scylla_example_with_keyspace:orders": 2, - } + # Confirm record masked in non-default schema + assert erasure_results == { + f"{database_name}:customer": 1, + f"{database_name}:payment_card": 0, + f"{database_name}:orders": 0, + f"{database_name}:address": 0, + }, "Only one record on customer table has targeted data category" + customer_records = postgres_integration_db.execute( + text("select * from backup_schema.customer where id = 1;") + ) + johanna_record = [c for c in customer_records][0] + assert johanna_record.email == customer_email # Not masked + assert johanna_record.name is None # Masked by erasure request +@pytest.mark.integration_postgres @pytest.mark.integration @pytest.mark.asyncio @pytest.mark.parametrize( @@ -1565,18 +1085,17 @@ async def test_retry_erasure( execution_logs = db.query(ExecutionLog).filter_by( privacy_request_id=privacy_request.id, action_type=ActionType.erasure ) - assert 40 == execution_logs.count() + assert 44 == execution_logs.count() - # These nodes were able to complete because they didn't have a PK - nothing to erase visit_logs = execution_logs.filter_by(collection_name="visit") - assert {"in_processing", "complete"} == { + assert ["in_processing", "retrying", "retrying", "error"] == [ el.status.value for el in visit_logs - } + ] order_item_logs = execution_logs.filter_by(collection_name="order_item") - assert {"in_processing", "complete"} == { + assert ["in_processing", "retrying", "retrying", "error"] == [ el.status.value for el in order_item_logs - } + ] # Address log mask data couldn't run, attempted to retry twice per configuration address_logs = execution_logs.filter_by(collection_name="address").order_by( ExecutionLog.created_at @@ -1585,297 +1104,19 @@ async def test_retry_erasure( el.status.value for el in address_logs ] - # Downstream request tasks were marked as error. Some tasks completed because there is no PK - # on their collection and we can't erase - assert {rt.status.value for rt in privacy_request.erasure_tasks} == { + # Downstream request tasks (other than __ROOT__) were marked as error. + assert [rt.status.value for rt in privacy_request.erasure_tasks] == [ "complete", "error", "error", "error", - "complete", "error", "error", "error", "error", "error", - "complete", "error", "error", - } - - -@pytest.mark.integration_timescale -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_timescale_access_request_task( - db, - policy, - timescale_connection_config, - timescale_integration_db, - privacy_request, - dsr_version, - request, -) -> None: - database_name = "my_timescale_db_1" - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - v = access_runner_tester( - privacy_request, - policy, - integration_db_graph(database_name), - [timescale_connection_config], - {"email": "customer-1@example.com"}, - db, - ) - - assert_rows_match( - v[f"{database_name}:address"], - min_size=2, - keys=["id", "street", "city", "state", "zip"], - ) - assert_rows_match( - v[f"{database_name}:orders"], - min_size=3, - keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], - ) - assert_rows_match( - v[f"{database_name}:payment_card"], - min_size=2, - keys=["id", "name", "ccn", "customer_id", "billing_address_id"], - ) - assert_rows_match( - v[f"{database_name}:customer"], - min_size=1, - keys=["id", "name", "email", "address_id"], - ) - - # links - assert v[f"{database_name}:customer"][0]["email"] == "customer-1@example.com" - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] - - assert ( - len( - records_matching_fields( - logs, dataset_name=database_name, collection_name="customer" - ) - ) - > 0 - ) - - assert ( - len( - records_matching_fields( - logs, dataset_name=database_name, collection_name="address" - ) - ) - > 0 - ) - - assert ( - len( - records_matching_fields( - logs, dataset_name=database_name, collection_name="orders" - ) - ) - > 0 - ) - - assert ( - len( - records_matching_fields( - logs, - dataset_name=database_name, - collection_name="payment_card", - ) - ) - > 0 - ) - - -@pytest.mark.integration_timescale -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_timescale_erasure_request_task( - db, - erasure_policy, - timescale_connection_config, - timescale_integration_db, - privacy_request_with_erasure_policy, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - rule = erasure_policy.rules[0] - target = rule.targets[0] - target.data_category = "user" - target.save(db) - - database_name = "my_timescale_db_1" - - dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) - - # Set some data categories on fields that will be targeted by the policy above - field([dataset], database_name, "customer", "name").data_categories = ["user.name"] - field([dataset], database_name, "address", "street").data_categories = ["user"] - field([dataset], database_name, "payment_card", "ccn").data_categories = ["user"] - - graph = DatasetGraph(dataset) - - v = access_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "customer-1@example.com"}, - db, - ) - - v = erasure_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "customer-1@example.com"}, - get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), - db, - ) - assert v == { - f"{database_name}:customer": 1, - f"{database_name}:orders": 0, - f"{database_name}:payment_card": 2, - f"{database_name}:address": 2, - }, "No erasure on orders table - no data categories targeted" - - # Verify masking in appropriate tables - address_cursor = timescale_integration_db.execute( - text("select * from address where id in (1, 2)") - ) - for address in address_cursor: - assert address.street is None # Masked due to matching data category - assert address.state is not None - assert address.city is not None - assert address.zip is not None - - customer_cursor = timescale_integration_db.execute( - text("select * from customer where id = 1") - ) - customer = [customer for customer in customer_cursor][0] - assert customer.name is None # Masked due to matching data category - assert customer.email == "customer-1@example.com" - assert customer.address_id is not None - - payment_card_cursor = timescale_integration_db.execute( - text("select * from payment_card where id in ('pay_aaa-aaa', 'pay_bbb-bbb')") - ) - payment_cards = [card for card in payment_card_cursor] - assert all( - [card.ccn is None for card in payment_cards] - ) # Masked due to matching data category - assert not any([card.name is None for card in payment_cards]) is None - - -@pytest.mark.integration_timescale -@pytest.mark.integration -@pytest.mark.asyncio -@pytest.mark.parametrize( - "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], -) -async def test_timescale_query_and_mask_hypertable( - db, - erasure_policy, - timescale_connection_config, - timescale_integration_db, - privacy_request_with_erasure_policy, - dsr_version, - request, -) -> None: - request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - - database_name = "my_timescale_db_1" - - dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) - # For this test, add a new collection to our standard dataset corresponding to the - # "onsite_personnel" timescale hypertable - onsite_personnel_collection = Collection( - name="onsite_personnel", - fields=[ - ScalarField( - name="responsible", data_type_converter=str_converter, identity="email" - ), - ScalarField( - name="time", data_type_converter=str_converter, primary_key=True - ), - ], - ) - - dataset.collections.append(onsite_personnel_collection) - graph = DatasetGraph(dataset) - rule = erasure_policy.rules[0] - target = rule.targets[0] - target.data_category = "user" - target.save(db) - # Update data category on responsible field - field( - [dataset], database_name, "onsite_personnel", "responsible" - ).data_categories = ["user.contact.email"] - - access_results = access_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "employee-1@example.com"}, - db, - ) - - # Demonstrate hypertable can be queried - assert access_results[f"{database_name}:onsite_personnel"] == [ - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 1, 9, 0)}, - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 2, 9, 0)}, - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 3, 9, 0)}, - {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 5, 9, 0)}, - ] - - # Run an erasure on the hypertable targeting the responsible field - v = erasure_runner_tester( - privacy_request_with_erasure_policy, - erasure_policy, - graph, - [timescale_connection_config], - {"email": "employee-1@example.com"}, - get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), - db, - ) - - assert v == { - f"{database_name}:customer": 0, - f"{database_name}:orders": 0, - f"{database_name}:payment_card": 0, - f"{database_name}:address": 0, - f"{database_name}:onsite_personnel": 4, - }, "onsite_personnel.responsible was the only targeted data category" - - personnel_records = timescale_integration_db.execute( - text("select * from onsite_personnel") - ) - for record in personnel_records: - assert ( - record.responsible != "employee-1@example.com" - ) # These emails have all been masked + "error", + "error", + ] diff --git a/tests/ops/integration_tests/test_timescale_task.py b/tests/ops/integration_tests/test_timescale_task.py new file mode 100644 index 0000000000..97af65ce65 --- /dev/null +++ b/tests/ops/integration_tests/test_timescale_task.py @@ -0,0 +1,294 @@ +from datetime import datetime + +import pytest +from sqlalchemy import text + +from fides.api.graph.config import Collection, ScalarField +from fides.api.graph.graph import DatasetGraph +from fides.api.models.privacy_request import ExecutionLog +from fides.api.task.graph_task import get_cached_data_for_erasures + +from ...conftest import access_runner_tester, erasure_runner_tester +from ..graph.graph_test_util import assert_rows_match, field, records_matching_fields +from ..task.traversal_data import ( + integration_db_graph, + postgres_db_graph_dataset, + str_converter, +) + + +@pytest.mark.integration_timescale +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_timescale_access_request_task( + db, + policy, + timescale_connection_config, + timescale_integration_db, + privacy_request, + dsr_version, + request, +) -> None: + database_name = "my_timescale_db_1" + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + v = access_runner_tester( + privacy_request, + policy, + integration_db_graph(database_name), + [timescale_connection_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert_rows_match( + v[f"{database_name}:address"], + min_size=2, + keys=["id", "street", "city", "state", "zip"], + ) + assert_rows_match( + v[f"{database_name}:orders"], + min_size=3, + keys=["id", "customer_id", "shipping_address_id", "payment_card_id"], + ) + assert_rows_match( + v[f"{database_name}:payment_card"], + min_size=2, + keys=["id", "name", "ccn", "customer_id", "billing_address_id"], + ) + assert_rows_match( + v[f"{database_name}:customer"], + min_size=1, + keys=["id", "name", "email", "address_id"], + ) + + # links + assert v[f"{database_name}:customer"][0]["email"] == "customer-1@example.com" + + logs = ( + ExecutionLog.query(db=db) + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() + ) + + logs = [log.__dict__ for log in logs] + + assert ( + len( + records_matching_fields( + logs, dataset_name=database_name, collection_name="customer" + ) + ) + > 0 + ) + + assert ( + len( + records_matching_fields( + logs, dataset_name=database_name, collection_name="address" + ) + ) + > 0 + ) + + assert ( + len( + records_matching_fields( + logs, dataset_name=database_name, collection_name="orders" + ) + ) + > 0 + ) + + assert ( + len( + records_matching_fields( + logs, + dataset_name=database_name, + collection_name="payment_card", + ) + ) + > 0 + ) + + +@pytest.mark.integration_timescale +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_timescale_erasure_request_task( + db, + erasure_policy, + timescale_connection_config, + timescale_integration_db, + privacy_request_with_erasure_policy, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = "user" + target.save(db) + + database_name = "my_timescale_db_1" + + dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) + + # Set some data categories on fields that will be targeted by the policy above + field([dataset], database_name, "customer", "name").data_categories = ["user.name"] + field([dataset], database_name, "address", "street").data_categories = ["user"] + field([dataset], database_name, "payment_card", "ccn").data_categories = ["user"] + + graph = DatasetGraph(dataset) + + v = access_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "customer-1@example.com"}, + db, + ) + + v = erasure_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "customer-1@example.com"}, + get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), + db, + ) + assert v == { + f"{database_name}:customer": 1, + f"{database_name}:orders": 0, + f"{database_name}:payment_card": 2, + f"{database_name}:address": 2, + }, "No erasure on orders table - no data categories targeted" + + # Verify masking in appropriate tables + address_cursor = timescale_integration_db.execute( + text("select * from address where id in (1, 2)") + ) + for address in address_cursor: + assert address.street is None # Masked due to matching data category + assert address.state is not None + assert address.city is not None + assert address.zip is not None + + customer_cursor = timescale_integration_db.execute( + text("select * from customer where id = 1") + ) + customer = [customer for customer in customer_cursor][0] + assert customer.name is None # Masked due to matching data category + assert customer.email == "customer-1@example.com" + assert customer.address_id is not None + + payment_card_cursor = timescale_integration_db.execute( + text("select * from payment_card where id in ('pay_aaa-aaa', 'pay_bbb-bbb')") + ) + payment_cards = [card for card in payment_card_cursor] + assert all( + [card.ccn is None for card in payment_cards] + ) # Masked due to matching data category + assert not any([card.name is None for card in payment_cards]) is None + + +@pytest.mark.integration_timescale +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0", "use_dsr_2_0"], +) +async def test_timescale_query_and_mask_hypertable( + db, + erasure_policy, + timescale_connection_config, + timescale_integration_db, + privacy_request_with_erasure_policy, + dsr_version, + request, +) -> None: + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + database_name = "my_timescale_db_1" + + dataset = postgres_db_graph_dataset(database_name, timescale_connection_config.key) + # For this test, add a new collection to our standard dataset corresponding to the + # "onsite_personnel" timescale hypertable + onsite_personnel_collection = Collection( + name="onsite_personnel", + fields=[ + ScalarField( + name="responsible", data_type_converter=str_converter, identity="email" + ), + ScalarField( + name="time", data_type_converter=str_converter, primary_key=True + ), + ], + ) + + dataset.collections.append(onsite_personnel_collection) + graph = DatasetGraph(dataset) + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = "user" + target.save(db) + # Update data category on responsible field + field( + [dataset], database_name, "onsite_personnel", "responsible" + ).data_categories = ["user.contact.email"] + + access_results = access_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "employee-1@example.com"}, + db, + ) + + # Demonstrate hypertable can be queried + assert access_results[f"{database_name}:onsite_personnel"] == [ + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 1, 9, 0)}, + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 2, 9, 0)}, + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 3, 9, 0)}, + {"responsible": "employee-1@example.com", "time": datetime(2022, 1, 5, 9, 0)}, + ] + + # Run an erasure on the hypertable targeting the responsible field + v = erasure_runner_tester( + privacy_request_with_erasure_policy, + erasure_policy, + graph, + [timescale_connection_config], + {"email": "employee-1@example.com"}, + get_cached_data_for_erasures(privacy_request_with_erasure_policy.id), + db, + ) + + assert v == { + f"{database_name}:customer": 0, + f"{database_name}:orders": 0, + f"{database_name}:payment_card": 0, + f"{database_name}:address": 0, + f"{database_name}:onsite_personnel": 4, + }, "onsite_personnel.responsible was the only targeted data category" + + personnel_records = timescale_integration_db.execute( + text("select * from onsite_personnel") + ) + for record in personnel_records: + assert ( + record.responsible != "employee-1@example.com" + ) # These emails have all been masked diff --git a/tests/ops/service/connectors/test_bigquery_connector.py b/tests/ops/service/connectors/test_bigquery_connector.py index a9524777fe..2e7bc3b075 100644 --- a/tests/ops/service/connectors/test_bigquery_connector.py +++ b/tests/ops/service/connectors/test_bigquery_connector.py @@ -129,7 +129,7 @@ def test_generate_update_partitioned_table( assert len(updates) == 2 assert ( str(updates[0]) - == "UPDATE `silken-precinct-284918.fidesopstest.customer` SET `name`=%(name:STRING)s WHERE `silken-precinct-284918.fidesopstest.customer`.`id` = %(id_1:INT64)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" + == "UPDATE `silken-precinct-284918.fidesopstest.customer` SET `name`=%(name:STRING)s WHERE `silken-precinct-284918.fidesopstest.customer`.`email` = %(email_1:STRING)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" ) def test_generate_delete_partitioned_table( @@ -158,7 +158,7 @@ def test_generate_delete_partitioned_table( assert len(deletes) == 2 assert ( str(deletes[0]) - == "DELETE FROM `silken-precinct-284918.fidesopstest.customer` WHERE `silken-precinct-284918.fidesopstest.customer`.`id` = %(id_1:INT64)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" + == "DELETE FROM `silken-precinct-284918.fidesopstest.customer` WHERE `silken-precinct-284918.fidesopstest.customer`.`email` = %(email_1:STRING)s AND `created` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `created` <= CURRENT_TIMESTAMP()" ) def test_retrieve_partitioned_data( diff --git a/tests/ops/service/connectors/test_bigquery_queryconfig.py b/tests/ops/service/connectors/test_bigquery_queryconfig.py index 06c51c5105..24a16517b6 100644 --- a/tests/ops/service/connectors/test_bigquery_queryconfig.py +++ b/tests/ops/service/connectors/test_bigquery_queryconfig.py @@ -196,7 +196,7 @@ def test_generate_delete_stmt( ) stmts = set(str(stmt) for stmt in delete_stmts) expected_stmts = { - "DELETE FROM `employee` WHERE `employee`.`id` = %(id_1:STRING)s" + "DELETE FROM `employee` WHERE `employee`.`address_id` = %(address_id_1:STRING)s AND `employee`.`email` = %(email_1:STRING)s" } assert stmts == expected_stmts @@ -289,6 +289,6 @@ def test_generate_namespaced_delete_stmt( ) stmts = set(str(stmt) for stmt in delete_stmts) expected_stmts = { - "DELETE FROM `silken-precinct-284918.fidesopstest.employee` WHERE `silken-precinct-284918.fidesopstest.employee`.`id` = %(id_1:STRING)s" + "DELETE FROM `silken-precinct-284918.fidesopstest.employee` WHERE `silken-precinct-284918.fidesopstest.employee`.`address_id` = %(address_id_1:STRING)s AND `silken-precinct-284918.fidesopstest.employee`.`email` = %(email_1:STRING)s" } assert stmts == expected_stmts diff --git a/tests/ops/service/connectors/test_dynamodb_query_config.py b/tests/ops/service/connectors/test_dynamodb_query_config.py new file mode 100644 index 0000000000..4591ae9385 --- /dev/null +++ b/tests/ops/service/connectors/test_dynamodb_query_config.py @@ -0,0 +1,129 @@ +from datetime import datetime, timezone + +import pytest +from boto3.dynamodb.types import TypeDeserializer +from fideslang.models import Dataset + +from fides.api.graph.config import CollectionAddress +from fides.api.graph.graph import DatasetGraph +from fides.api.graph.traversal import Traversal +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.models.privacy_request import PrivacyRequest +from fides.api.service.connectors.query_configs.dynamodb_query_config import ( + DynamoDBQueryConfig, +) + +privacy_request = PrivacyRequest(id="234544") + + +class TestDynamoDBQueryConfig: + @pytest.fixture(scope="function") + def identity(self): + identity = {"email": "customer-test_uuid@example.com"} + return identity + + @pytest.fixture(scope="function") + def dataset_graph(self, integration_dynamodb_config, example_datasets): + dataset = Dataset(**example_datasets[11]) + dataset_graph = convert_dataset_to_graph( + dataset, integration_dynamodb_config.key + ) + + return DatasetGraph(*[dataset_graph]) + + @pytest.fixture(scope="function") + def traversal(self, identity, dataset_graph): + dynamo_traversal = Traversal(dataset_graph, identity) + return dynamo_traversal + + @pytest.fixture(scope="function") + def customer_node(self, traversal): + return traversal.traversal_node_dict[ + CollectionAddress("dynamodb_example_test_dataset", "customer") + ].to_mock_execution_node() + + @pytest.fixture(scope="function") + def customer_identifier_node(self, traversal): + return traversal.traversal_node_dict[ + CollectionAddress("dynamodb_example_test_dataset", "customer_identifier") + ].to_mock_execution_node() + + @pytest.fixture(scope="function") + def customer_row(self): + row = { + "customer_email": {"S": "customer-1@example.com"}, + "name": {"S": "John Customer"}, + "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, + "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, + "id": {"S": "1"}, + } + return row + + @pytest.fixture(scope="function") + def deserialized_customer_row(self, customer_row): + deserialized_customer_row = {} + deserializer = TypeDeserializer() + for key, value in customer_row.items(): + deserialized_customer_row[key] = deserializer.deserialize(value) + return deserialized_customer_row + + @pytest.fixture(scope="function") + def customer_identifier_row(self): + row = { + "customer_id": {"S": "customer-1@example.com"}, + "email": {"S": "customer-1@example.com"}, + "name": {"S": "Customer 1"}, + "created": {"S": datetime.now(timezone.utc).isoformat()}, + } + return row + + @pytest.fixture(scope="function") + def deserialized_customer_identifier_row(self, customer_identifier_row): + deserialized_customer_identifier_row = {} + deserializer = TypeDeserializer() + for key, value in customer_identifier_row.items(): + deserialized_customer_identifier_row[key] = deserializer.deserialize(value) + return deserialized_customer_identifier_row + + def test_get_query_param_formatting_single_key( + self, + resources_dict, + customer_node, + ) -> None: + input_data = { + "fidesops_grouped_inputs": [], + "email": ["customer-test_uuid@example.com"], + } + attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] + query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) + item = query_config.generate_query( + input_data=input_data, policy=resources_dict["policy"] + ) + assert item["ExpressionAttributeValues"] == { + ":value": {"S": "customer-test_uuid@example.com"} + } + assert item["KeyConditionExpression"] == "email = :value" + + def test_put_query_param_formatting_single_key( + self, + erasure_policy, + customer_node, + deserialized_customer_row, + ) -> None: + input_data = { + "fidesops_grouped_inputs": [], + "email": ["customer-test_uuid@example.com"], + } + attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] + query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) + update_item = query_config.generate_update_stmt( + deserialized_customer_row, erasure_policy, privacy_request + ) + + assert update_item == { + "customer_email": {"S": "customer-1@example.com"}, + "name": {"NULL": True}, + "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, + "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, + "id": {"S": "1"}, + } diff --git a/tests/ops/service/connectors/test_mongo_query_config.py b/tests/ops/service/connectors/test_mongo_query_config.py new file mode 100644 index 0000000000..c0f6079df1 --- /dev/null +++ b/tests/ops/service/connectors/test_mongo_query_config.py @@ -0,0 +1,283 @@ +import pytest +from fideslang.models import Dataset + +from fides.api.graph.config import ( + CollectionAddress, + FieldAddress, + FieldPath, + ObjectField, + ScalarField, +) +from fides.api.graph.graph import DatasetGraph, Edge +from fides.api.graph.traversal import Traversal +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.models.privacy_request import PrivacyRequest +from fides.api.schemas.masking.masking_configuration import HashMaskingConfiguration +from fides.api.schemas.masking.masking_secrets import MaskingSecretCache, SecretType +from fides.api.service.connectors.query_configs.mongodb_query_config import ( + MongoQueryConfig, +) +from fides.api.service.masking.strategy.masking_strategy_hash import HashMaskingStrategy +from fides.api.util.data_category import DataCategory + +from ...task.traversal_data import combined_mongo_postgresql_graph +from ...test_helpers.cache_secrets_helper import cache_secret + +privacy_request = PrivacyRequest(id="234544") + + +class TestMongoQueryConfig: + @pytest.fixture(scope="function") + def combined_traversal(self, connection_config, integration_mongodb_config): + mongo_dataset, postgres_dataset = combined_mongo_postgresql_graph( + connection_config, integration_mongodb_config + ) + combined_dataset_graph = DatasetGraph(mongo_dataset, postgres_dataset) + combined_traversal = Traversal( + combined_dataset_graph, + {"email": "customer-1@examplecom"}, + ) + return combined_traversal + + @pytest.fixture(scope="function") + def customer_details_node(self, combined_traversal): + return combined_traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + + @pytest.fixture(scope="function") + def customer_feedback_node(self, combined_traversal): + return combined_traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_feedback") + ].to_mock_execution_node() + + def test_field_map_nested(self, customer_details_node): + config = MongoQueryConfig(customer_details_node) + + field_map = config.field_map() + assert isinstance(field_map[FieldPath("workplace_info")], ObjectField) + assert isinstance( + field_map[FieldPath("workplace_info", "employer")], ScalarField + ) + + def test_primary_key_field_paths(self, customer_details_node): + config = MongoQueryConfig(customer_details_node) + assert list(config.primary_key_field_paths.keys()) == [FieldPath("_id")] + assert isinstance(config.primary_key_field_paths[FieldPath("_id")], ScalarField) + + def test_nested_query_field_paths( + self, customer_details_node, customer_feedback_node + ): + assert customer_details_node.query_field_paths == { + FieldPath("customer_id"), + } + + assert customer_feedback_node.query_field_paths == { + FieldPath("customer_information", "email") + } + + def test_nested_typed_filtered_values(self, customer_feedback_node): + """Identity data is located on a nested object""" + input_data = { + "customer_information.email": ["test@example.com"], + "ignore": ["abcde"], + } + assert customer_feedback_node.typed_filtered_values(input_data) == { + "customer_information.email": ["test@example.com"] + } + + def test_generate_query( + self, + policy, + example_datasets, + integration_mongodb_config, + connection_config, + ): + dataset_postgres = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) + dataset_mongo = Dataset(**example_datasets[1]) + mongo_graph = convert_dataset_to_graph( + dataset_mongo, integration_mongodb_config.key + ) + dataset_graph = DatasetGraph(*[graph, mongo_graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + # Edge created from Root to nested customer_information.email field + assert ( + Edge( + FieldAddress("__ROOT__", "__ROOT__", "email"), + FieldAddress( + "mongo_test", "customer_feedback", "customer_information", "email" + ), + ) + in traversal.edges + ) + + # Test query on nested field + customer_feedback = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_feedback") + ].to_mock_execution_node() + config = MongoQueryConfig(customer_feedback) + input_data = {"customer_information.email": ["customer-1@example.com"]} + # Tuple of query, projection - Searching for documents with nested + # customer_information.email = customer-1@example.com + assert config.generate_query(input_data, policy) == ( + {"customer_information.email": "customer-1@example.com"}, + {"_id": 1, "customer_information": 1, "date": 1, "message": 1, "rating": 1}, + ) + + # Test query nested data + customer_details = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + config = MongoQueryConfig(customer_details) + input_data = {"customer_id": [1]} + # Tuple of query, projection - Projection is specifying fields at the top-level. Nested data will + # be filtered later. + assert config.generate_query(input_data, policy) == ( + {"customer_id": 1}, + { + "_id": 1, + "birthday": 1, + "comments": 1, + "customer_id": 1, + "customer_uuid": 1, + "emergency_contacts": 1, + "children": 1, + "gender": 1, + "travel_identifiers": 1, + "workplace_info": 1, + }, + ) + + def test_generate_update_stmt_multiple_fields( + self, + erasure_policy, + example_datasets, + integration_mongodb_config, + connection_config, + ): + dataset_postgres = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) + dataset_mongo = Dataset(**example_datasets[1]) + mongo_graph = convert_dataset_to_graph( + dataset_mongo, integration_mongodb_config.key + ) + dataset_graph = DatasetGraph(*[graph, mongo_graph]) + + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + customer_details = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + config = MongoQueryConfig(customer_details) + row = { + "birthday": "1988-01-10", + "gender": "male", + "customer_id": 1, + "_id": 1, + "workplace_info": { + "position": "Chief Strategist", + "direct_reports": ["Robbie Margo", "Sully Hunter"], + }, + "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], + "children": ["Christopher Customer", "Courtney Customer"], + } + + # Make target more broad + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = DataCategory("user").value + + mongo_statement = config.generate_update_stmt( + row, erasure_policy, privacy_request + ) + + expected_result_0 = {"_id": 1} + expected_result_1 = { + "$set": { + "birthday": None, + "children.0": None, + "children.1": None, + "customer_id": None, + "emergency_contacts.0.name": None, + "workplace_info.direct_reports.0": None, # Both direct reports are masked. + "workplace_info.direct_reports.1": None, + "emergency_contacts.0.phone": None, + "gender": None, + "workplace_info.position": None, + } + } + + print(mongo_statement[1]) + print(expected_result_1) + assert mongo_statement[0] == expected_result_0 + assert mongo_statement[1] == expected_result_1 + + def test_generate_update_stmt_multiple_rules( + self, + erasure_policy_two_rules, + example_datasets, + integration_mongodb_config, + connection_config, + ): + dataset_postgres = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) + dataset_mongo = Dataset(**example_datasets[1]) + mongo_graph = convert_dataset_to_graph( + dataset_mongo, integration_mongodb_config.key + ) + dataset_graph = DatasetGraph(*[graph, mongo_graph]) + + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + + customer_details = traversal.traversal_node_dict[ + CollectionAddress("mongo_test", "customer_details") + ].to_mock_execution_node() + + config = MongoQueryConfig(customer_details) + row = { + "birthday": "1988-01-10", + "gender": "male", + "customer_id": 1, + "_id": 1, + "workplace_info": { + "position": "Chief Strategist", + "direct_reports": ["Robbie Margo", "Sully Hunter"], + }, + "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], + "children": ["Christopher Customer", "Courtney Customer"], + } + + rule = erasure_policy_two_rules.rules[0] + rule.masking_strategy = { + "strategy": "hash", + "configuration": {"algorithm": "SHA-512"}, + } + target = rule.targets[0] + target.data_category = DataCategory("user.demographic.date_of_birth").value + + rule_two = erasure_policy_two_rules.rules[1] + rule_two.masking_strategy = { + "strategy": "random_string_rewrite", + "configuration": {"length": 30}, + } + target = rule_two.targets[0] + target.data_category = DataCategory("user.demographic.gender").value + # cache secrets for hash strategy + secret = MaskingSecretCache[str]( + secret="adobo", + masking_strategy=HashMaskingStrategy.name, + secret_type=SecretType.salt, + ) + cache_secret(secret, privacy_request.id) + + mongo_statement = config.generate_update_stmt( + row, erasure_policy_two_rules, privacy_request + ) + assert mongo_statement[0] == {"_id": 1} + assert len(mongo_statement[1]["$set"]["gender"]) == 30 + assert ( + mongo_statement[1]["$set"]["birthday"] + == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( + ["1988-01-10"], request_id=privacy_request.id + )[0] + ) diff --git a/tests/ops/service/connectors/test_query_config.py b/tests/ops/service/connectors/test_query_config.py index 01d7b9dbd2..eac650d587 100644 --- a/tests/ops/service/connectors/test_query_config.py +++ b/tests/ops/service/connectors/test_query_config.py @@ -1,43 +1,29 @@ -from datetime import datetime, timezone from typing import Any, Dict, Set from unittest import mock import pytest -from boto3.dynamodb.types import TypeDeserializer from fideslang.models import Dataset from fides.api.common_exceptions import MissingNamespaceSchemaException -from fides.api.graph.config import ( - CollectionAddress, - FieldAddress, - FieldPath, - ObjectField, - ScalarField, -) +from fides.api.graph.config import CollectionAddress, FieldPath from fides.api.graph.execution import ExecutionNode -from fides.api.graph.graph import DatasetGraph, Edge +from fides.api.graph.graph import DatasetGraph from fides.api.graph.traversal import Traversal, TraversalNode from fides.api.models.datasetconfig import convert_dataset_to_graph from fides.api.models.privacy_request import PrivacyRequest from fides.api.schemas.masking.masking_configuration import HashMaskingConfiguration from fides.api.schemas.masking.masking_secrets import MaskingSecretCache, SecretType from fides.api.schemas.namespace_meta.namespace_meta import NamespaceMeta -from fides.api.service.connectors.query_configs.dynamodb_query_config import ( - DynamoDBQueryConfig, -) -from fides.api.service.connectors.query_configs.mongodb_query_config import ( - MongoQueryConfig, -) from fides.api.service.connectors.query_configs.query_config import ( QueryConfig, SQLQueryConfig, ) -from fides.api.service.connectors.scylla_query_config import ScyllaDBQueryConfig from fides.api.service.masking.strategy.masking_strategy_hash import HashMaskingStrategy from fides.api.util.data_category import DataCategory from tests.fixtures.application_fixtures import load_dataset +from tests.ops.test_helpers.dataset_utils import remove_primary_keys -from ...task.traversal_data import combined_mongo_postgresql_graph, integration_db_graph +from ...task.traversal_data import integration_db_graph from ...test_helpers.cache_secrets_helper import cache_secret, clear_cache_secrets # customers -> address, order @@ -286,9 +272,47 @@ def test_generate_update_stmt_one_field( "id": 1, } text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) - assert text_clause.text == """UPDATE customer SET name = :name WHERE id = :id""" - assert text_clause._bindparams["name"].key == "name" - assert text_clause._bindparams["name"].value is None # Null masking strategy + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE id = :id""" + ) + assert text_clause._bindparams["masked_name"].key == "masked_name" + assert ( + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy + + def test_generate_update_stmt_one_field_inbound_reference( + self, erasure_policy_address_city, example_datasets, connection_config + ): + dataset = Dataset(**example_datasets[0]) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) + + address_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "address") + ].to_mock_execution_node() + + config = SQLQueryConfig(address_node) + row = { + "id": 1, + "house": "123", + "street": "Main St", + "city": "San Francisco", + "state": "CA", + "zip": "94105", + } + text_clause = config.generate_update_stmt( + row, erasure_policy_address_city, privacy_request + ) + assert ( + text_clause.text + == """UPDATE address SET city = :masked_city WHERE id = :id""" + ) + assert text_clause._bindparams["masked_city"].key == "masked_city" + assert ( + text_clause._bindparams["masked_city"].value is None + ) # Null masking strategy def test_generate_update_stmt_length_truncation( self, @@ -316,11 +340,14 @@ def test_generate_update_stmt_length_truncation( text_clause = config.generate_update_stmt( row, erasure_policy_string_rewrite_long, privacy_request ) - assert text_clause.text == """UPDATE customer SET name = :name WHERE id = :id""" - assert text_clause._bindparams["name"].key == "name" + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE id = :id""" + ) + assert text_clause._bindparams["masked_name"].key == "masked_name" # length truncation on name field assert ( - text_clause._bindparams["name"].value + text_clause._bindparams["masked_name"].value == "some rewrite value that is very long and" ) @@ -365,22 +392,23 @@ def test_generate_update_stmt_multiple_fields_same_rule( text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) assert ( text_clause.text - == "UPDATE customer SET email = :email, name = :name WHERE id = :id" + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE id = :id" ) - assert text_clause._bindparams["name"].key == "name" + assert text_clause._bindparams["masked_name"].key == "masked_name" # since length is set to 40 in dataset.yml, we expect only first 40 chars of masked val assert ( - text_clause._bindparams["name"].value + text_clause._bindparams["masked_name"].value == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( ["John Customer"], request_id=privacy_request.id )[0][0:40] ) assert ( - text_clause._bindparams["email"].value + text_clause._bindparams["masked_email"].value == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( ["customer-1@example.com"], request_id=privacy_request.id )[0] ) + assert text_clause._bindparams["id"].value == 1 clear_cache_secrets(privacy_request.id) def test_generate_update_stmts_from_multiple_rules( @@ -409,251 +437,145 @@ def test_generate_update_stmts_from_multiple_rules( assert ( text_clause.text - == "UPDATE customer SET email = :email, name = :name WHERE id = :id" + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE id = :id" ) # Two different masking strategies used for name and email - assert text_clause._bindparams["name"].value is None # Null masking strategy assert ( - text_clause._bindparams["email"].value == "*****" + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy + assert ( + text_clause._bindparams["masked_email"].value == "*****" ) # String rewrite masking strategy + def test_generate_update_stmt_one_field_without_primary_keys( + self, erasure_policy, example_datasets, connection_config + ): + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) -class TestMongoQueryConfig: - @pytest.fixture(scope="function") - def combined_traversal(self, connection_config, integration_mongodb_config): - mongo_dataset, postgres_dataset = combined_mongo_postgresql_graph( - connection_config, integration_mongodb_config - ) - combined_dataset_graph = DatasetGraph(mongo_dataset, postgres_dataset) - combined_traversal = Traversal( - combined_dataset_graph, - {"email": "customer-1@examplecom"}, - ) - return combined_traversal - - @pytest.fixture(scope="function") - def customer_details_node(self, combined_traversal): - return combined_traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") - ].to_mock_execution_node() - - @pytest.fixture(scope="function") - def customer_feedback_node(self, combined_traversal): - return combined_traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_feedback") + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - def test_field_map_nested(self, customer_details_node): - config = MongoQueryConfig(customer_details_node) - - field_map = config.field_map() - assert isinstance(field_map[FieldPath("workplace_info")], ObjectField) - assert isinstance( - field_map[FieldPath("workplace_info", "employer")], ScalarField + config = SQLQueryConfig(customer_node) + row = { + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, + } + text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE email = :email""" ) + assert text_clause._bindparams["masked_name"].key == "masked_name" + assert ( + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy - def test_primary_key_field_paths(self, customer_details_node): - config = MongoQueryConfig(customer_details_node) - assert list(config.primary_key_field_paths.keys()) == [FieldPath("_id")] - assert isinstance(config.primary_key_field_paths[FieldPath("_id")], ScalarField) - - def test_nested_query_field_paths( - self, customer_details_node, customer_feedback_node + def test_generate_update_stmt_one_field_inbound_reference_without_primary_keys( + self, erasure_policy_address_city, example_datasets, connection_config ): - assert customer_details_node.query_field_paths == { - FieldPath("customer_id"), - } + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - assert customer_feedback_node.query_field_paths == { - FieldPath("customer_information", "email") - } + address_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "address") + ].to_mock_execution_node() - def test_nested_typed_filtered_values(self, customer_feedback_node): - """Identity data is located on a nested object""" - input_data = { - "customer_information.email": ["test@example.com"], - "ignore": ["abcde"], - } - assert customer_feedback_node.typed_filtered_values(input_data) == { - "customer_information.email": ["test@example.com"] + config = SQLQueryConfig(address_node) + row = { + "id": 1, + "house": "123", + "street": "Main St", + "city": "San Francisco", + "state": "CA", + "zip": "94105", } - - def test_generate_query( - self, - policy, - example_datasets, - integration_mongodb_config, - connection_config, - ): - dataset_postgres = Dataset(**example_datasets[0]) - graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) - dataset_mongo = Dataset(**example_datasets[1]) - mongo_graph = convert_dataset_to_graph( - dataset_mongo, integration_mongodb_config.key + text_clause = config.generate_update_stmt( + row, erasure_policy_address_city, privacy_request ) - dataset_graph = DatasetGraph(*[graph, mongo_graph]) - traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - # Edge created from Root to nested customer_information.email field assert ( - Edge( - FieldAddress("__ROOT__", "__ROOT__", "email"), - FieldAddress( - "mongo_test", "customer_feedback", "customer_information", "email" - ), - ) - in traversal.edges - ) - - # Test query on nested field - customer_feedback = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_feedback") - ].to_mock_execution_node() - config = MongoQueryConfig(customer_feedback) - input_data = {"customer_information.email": ["customer-1@example.com"]} - # Tuple of query, projection - Searching for documents with nested - # customer_information.email = customer-1@example.com - assert config.generate_query(input_data, policy) == ( - {"customer_information.email": "customer-1@example.com"}, - {"_id": 1, "customer_information": 1, "date": 1, "message": 1, "rating": 1}, - ) - - # Test query nested data - customer_details = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") - ].to_mock_execution_node() - config = MongoQueryConfig(customer_details) - input_data = {"customer_id": [1]} - # Tuple of query, projection - Projection is specifying fields at the top-level. Nested data will - # be filtered later. - assert config.generate_query(input_data, policy) == ( - {"customer_id": 1}, - { - "_id": 1, - "birthday": 1, - "comments": 1, - "customer_id": 1, - "customer_uuid": 1, - "emergency_contacts": 1, - "children": 1, - "gender": 1, - "travel_identifiers": 1, - "workplace_info": 1, - }, + text_clause.text + == """UPDATE address SET city = :masked_city WHERE id = :id""" ) + assert text_clause._bindparams["masked_city"].key == "masked_city" + assert ( + text_clause._bindparams["masked_city"].value is None + ) # Null masking strategy - def test_generate_update_stmt_multiple_fields( + def test_generate_update_stmt_length_truncation_without_primary_keys( self, - erasure_policy, + erasure_policy_string_rewrite_long, example_datasets, - integration_mongodb_config, connection_config, ): - dataset_postgres = Dataset(**example_datasets[0]) - graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) - dataset_mongo = Dataset(**example_datasets[1]) - mongo_graph = convert_dataset_to_graph( - dataset_mongo, integration_mongodb_config.key - ) - dataset_graph = DatasetGraph(*[graph, mongo_graph]) - + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - customer_details = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") + + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - config = MongoQueryConfig(customer_details) + + config = SQLQueryConfig(customer_node) row = { - "birthday": "1988-01-10", - "gender": "male", - "customer_id": 1, - "_id": 1, - "workplace_info": { - "position": "Chief Strategist", - "direct_reports": ["Robbie Margo", "Sully Hunter"], - }, - "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], - "children": ["Christopher Customer", "Courtney Customer"], + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, } - # Make target more broad - rule = erasure_policy.rules[0] - target = rule.targets[0] - target.data_category = DataCategory("user").value - - mongo_statement = config.generate_update_stmt( - row, erasure_policy, privacy_request + text_clause = config.generate_update_stmt( + row, erasure_policy_string_rewrite_long, privacy_request ) - - expected_result_0 = {"_id": 1} - expected_result_1 = { - "$set": { - "birthday": None, - "children.0": None, - "children.1": None, - "customer_id": None, - "emergency_contacts.0.name": None, - "workplace_info.direct_reports.0": None, # Both direct reports are masked. - "workplace_info.direct_reports.1": None, - "emergency_contacts.0.phone": None, - "gender": None, - "workplace_info.position": None, - } - } - - print(mongo_statement[1]) - print(expected_result_1) - assert mongo_statement[0] == expected_result_0 - assert mongo_statement[1] == expected_result_1 - - def test_generate_update_stmt_multiple_rules( - self, - erasure_policy_two_rules, - example_datasets, - integration_mongodb_config, - connection_config, - ): - dataset_postgres = Dataset(**example_datasets[0]) - graph = convert_dataset_to_graph(dataset_postgres, connection_config.key) - dataset_mongo = Dataset(**example_datasets[1]) - mongo_graph = convert_dataset_to_graph( - dataset_mongo, integration_mongodb_config.key + assert ( + text_clause.text + == """UPDATE customer SET name = :masked_name WHERE email = :email""" + ) + assert text_clause._bindparams["masked_name"].key == "masked_name" + # length truncation on name field + assert ( + text_clause._bindparams["masked_name"].value + == "some rewrite value that is very long and" ) - dataset_graph = DatasetGraph(*[graph, mongo_graph]) + def test_generate_update_stmt_multiple_fields_same_rule_without_primary_keys( + self, erasure_policy, example_datasets, connection_config + ): + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) - customer_details = traversal.traversal_node_dict[ - CollectionAddress("mongo_test", "customer_details") + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - config = MongoQueryConfig(customer_details) + config = SQLQueryConfig(customer_node) row = { - "birthday": "1988-01-10", - "gender": "male", - "customer_id": 1, - "_id": 1, - "workplace_info": { - "position": "Chief Strategist", - "direct_reports": ["Robbie Margo", "Sully Hunter"], - }, - "emergency_contacts": [{"name": "June Customer", "phone": "444-444-4444"}], - "children": ["Christopher Customer", "Courtney Customer"], + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, } - rule = erasure_policy_two_rules.rules[0] + # Make target more broad + rule = erasure_policy.rules[0] + target = rule.targets[0] + target.data_category = DataCategory("user").value + + # Update rule masking strategy rule.masking_strategy = { "strategy": "hash", "configuration": {"algorithm": "SHA-512"}, } - target = rule.targets[0] - target.data_category = DataCategory("user.demographic.date_of_birth").value - - rule_two = erasure_policy_two_rules.rules[1] - rule_two.masking_strategy = { - "strategy": "random_string_rewrite", - "configuration": {"length": 30}, - } - target = rule_two.targets[0] - target.data_category = DataCategory("user.demographic.gender").value # cache secrets for hash strategy secret = MaskingSecretCache[str]( secret="adobo", @@ -662,169 +584,63 @@ def test_generate_update_stmt_multiple_rules( ) cache_secret(secret, privacy_request.id) - mongo_statement = config.generate_update_stmt( - row, erasure_policy_two_rules, privacy_request + text_clause = config.generate_update_stmt(row, erasure_policy, privacy_request) + assert ( + text_clause.text + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE email = :email" ) - assert mongo_statement[0] == {"_id": 1} - assert len(mongo_statement[1]["$set"]["gender"]) == 30 + assert text_clause._bindparams["masked_name"].key == "masked_name" + # since length is set to 40 in dataset.yml, we expect only first 40 chars of masked val assert ( - mongo_statement[1]["$set"]["birthday"] + text_clause._bindparams["masked_name"].value == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( - ["1988-01-10"], request_id=privacy_request.id - )[0] + ["John Customer"], request_id=privacy_request.id + )[0][0:40] ) - - -class TestDynamoDBQueryConfig: - @pytest.fixture(scope="function") - def identity(self): - identity = {"email": "customer-test_uuid@example.com"} - return identity - - @pytest.fixture(scope="function") - def dataset_graph(self, integration_dynamodb_config, example_datasets): - dataset = Dataset(**example_datasets[11]) - dataset_graph = convert_dataset_to_graph( - dataset, integration_dynamodb_config.key + assert ( + text_clause._bindparams["masked_email"].value + == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( + ["customer-1@example.com"], request_id=privacy_request.id + )[0] ) + assert text_clause._bindparams["email"].value == "customer-1@example.com" + clear_cache_secrets(privacy_request.id) - return DatasetGraph(*[dataset_graph]) - - @pytest.fixture(scope="function") - def traversal(self, identity, dataset_graph): - dynamo_traversal = Traversal(dataset_graph, identity) - return dynamo_traversal - - @pytest.fixture(scope="function") - def customer_node(self, traversal): - return traversal.traversal_node_dict[ - CollectionAddress("dynamodb_example_test_dataset", "customer") - ].to_mock_execution_node() - - @pytest.fixture(scope="function") - def customer_identifier_node(self, traversal): - return traversal.traversal_node_dict[ - CollectionAddress("dynamodb_example_test_dataset", "customer_identifier") - ].to_mock_execution_node() - - @pytest.fixture(scope="function") - def customer_row(self): - row = { - "customer_email": {"S": "customer-1@example.com"}, - "name": {"S": "John Customer"}, - "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, - "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, - "id": {"S": "1"}, - } - return row - - @pytest.fixture(scope="function") - def deserialized_customer_row(self, customer_row): - deserialized_customer_row = {} - deserializer = TypeDeserializer() - for key, value in customer_row.items(): - deserialized_customer_row[key] = deserializer.deserialize(value) - return deserialized_customer_row - - @pytest.fixture(scope="function") - def customer_identifier_row(self): + def test_generate_update_stmts_from_multiple_rules_without_primary_keys( + self, erasure_policy_two_rules, example_datasets, connection_config + ): + dataset = remove_primary_keys(Dataset(**example_datasets[0])) + graph = convert_dataset_to_graph(dataset, connection_config.key) + dataset_graph = DatasetGraph(*[graph]) + traversal = Traversal(dataset_graph, {"email": "customer-1@example.com"}) row = { - "customer_id": {"S": "customer-1@example.com"}, - "email": {"S": "customer-1@example.com"}, - "name": {"S": "Customer 1"}, - "created": {"S": datetime.now(timezone.utc).isoformat()}, - } - return row - - @pytest.fixture(scope="function") - def deserialized_customer_identifier_row(self, customer_identifier_row): - deserialized_customer_identifier_row = {} - deserializer = TypeDeserializer() - for key, value in customer_identifier_row.items(): - deserialized_customer_identifier_row[key] = deserializer.deserialize(value) - return deserialized_customer_identifier_row - - def test_get_query_param_formatting_single_key( - self, - resources_dict, - customer_node, - ) -> None: - input_data = { - "fidesops_grouped_inputs": [], - "email": ["customer-test_uuid@example.com"], - } - attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] - query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) - item = query_config.generate_query( - input_data=input_data, policy=resources_dict["policy"] - ) - assert item["ExpressionAttributeValues"] == { - ":value": {"S": "customer-test_uuid@example.com"} - } - assert item["KeyConditionExpression"] == "email = :value" - - def test_put_query_param_formatting_single_key( - self, - erasure_policy, - customer_node, - deserialized_customer_row, - ) -> None: - input_data = { - "fidesops_grouped_inputs": [], - "email": ["customer-test_uuid@example.com"], - } - attribute_definitions = [{"AttributeName": "email", "AttributeType": "S"}] - query_config = DynamoDBQueryConfig(customer_node, attribute_definitions) - update_item = query_config.generate_update_stmt( - deserialized_customer_row, erasure_policy, privacy_request - ) - - assert update_item == { - "customer_email": {"S": "customer-1@example.com"}, - "name": {"NULL": True}, - "address_id": {"L": [{"S": "1"}, {"S": "2"}]}, - "personal_info": {"M": {"gender": {"S": "male"}, "age": {"S": "99"}}}, - "id": {"S": "1"}, + "email": "customer-1@example.com", + "name": "John Customer", + "address_id": 1, + "id": 1, } - -class TestScyllaDBQueryConfig: - @pytest.fixture(scope="function") - def complete_execution_node( - self, example_datasets, integration_scylladb_config_with_keyspace - ): - dataset = Dataset(**example_datasets[15]) - graph = convert_dataset_to_graph( - dataset, integration_scylladb_config_with_keyspace.key - ) - dataset_graph = DatasetGraph(*[graph]) - identity = {"email": "customer-1@example.com"} - scylla_traversal = Traversal(dataset_graph, identity) - return scylla_traversal.traversal_node_dict[ - CollectionAddress("scylladb_example_test_dataset", "users") + customer_node = traversal.traversal_node_dict[ + CollectionAddress("postgres_example_test_dataset", "customer") ].to_mock_execution_node() - def test_dry_run_query_no_data(self, scylladb_execution_node): - query_config = ScyllaDBQueryConfig(scylladb_execution_node) - dry_run_query = query_config.dry_run_query() - assert dry_run_query is None + config = SQLQueryConfig(customer_node) - def test_dry_run_query_with_data(self, complete_execution_node): - query_config = ScyllaDBQueryConfig(complete_execution_node) - dry_run_query = query_config.dry_run_query() - assert ( - dry_run_query - == "SELECT age, alternative_contacts, ascii_data, big_int_data, do_not_contact, double_data, duration, email, float_data, last_contacted, logins, name, states_lived, timestamp, user_id, uuid FROM users WHERE email = ? ALLOW FILTERING;" + text_clause = config.generate_update_stmt( + row, erasure_policy_two_rules, privacy_request ) - def test_query_to_str(self, complete_execution_node): - query_config = ScyllaDBQueryConfig(complete_execution_node) - statement = ( - "SELECT name FROM users WHERE email = %(email)s", - {"email": "test@example.com"}, + assert ( + text_clause.text + == "UPDATE customer SET email = :masked_email, name = :masked_name WHERE email = :email" ) - query_to_str = query_config.query_to_str(statement, {}) - assert query_to_str == "SELECT name FROM users WHERE email = 'test@example.com'" + # Two different masking strategies used for name and email + assert ( + text_clause._bindparams["masked_name"].value is None + ) # Null masking strategy + assert ( + text_clause._bindparams["masked_email"].value == "*****" + ) # String rewrite masking strategy class TestSQLLikeQueryConfig: diff --git a/tests/ops/service/connectors/test_scylladb_query_config.py b/tests/ops/service/connectors/test_scylladb_query_config.py new file mode 100644 index 0000000000..3cbc6f493f --- /dev/null +++ b/tests/ops/service/connectors/test_scylladb_query_config.py @@ -0,0 +1,47 @@ +import pytest +from fideslang.models import Dataset + +from fides.api.graph.config import CollectionAddress +from fides.api.graph.graph import DatasetGraph +from fides.api.graph.traversal import Traversal +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.service.connectors.scylla_query_config import ScyllaDBQueryConfig + + +class TestScyllaDBQueryConfig: + @pytest.fixture(scope="function") + def complete_execution_node( + self, example_datasets, integration_scylladb_config_with_keyspace + ): + dataset = Dataset(**example_datasets[15]) + graph = convert_dataset_to_graph( + dataset, integration_scylladb_config_with_keyspace.key + ) + dataset_graph = DatasetGraph(*[graph]) + identity = {"email": "customer-1@example.com"} + scylla_traversal = Traversal(dataset_graph, identity) + return scylla_traversal.traversal_node_dict[ + CollectionAddress("scylladb_example_test_dataset", "users") + ].to_mock_execution_node() + + def test_dry_run_query_no_data(self, scylladb_execution_node): + query_config = ScyllaDBQueryConfig(scylladb_execution_node) + dry_run_query = query_config.dry_run_query() + assert dry_run_query is None + + def test_dry_run_query_with_data(self, complete_execution_node): + query_config = ScyllaDBQueryConfig(complete_execution_node) + dry_run_query = query_config.dry_run_query() + assert ( + dry_run_query + == "SELECT age, alternative_contacts, ascii_data, big_int_data, do_not_contact, double_data, duration, email, float_data, last_contacted, logins, name, states_lived, timestamp, user_id, uuid FROM users WHERE email = ? ALLOW FILTERING;" + ) + + def test_query_to_str(self, complete_execution_node): + query_config = ScyllaDBQueryConfig(complete_execution_node) + statement = ( + "SELECT name FROM users WHERE email = %(email)s", + {"email": "test@example.com"}, + ) + query_to_str = query_config.query_to_str(statement, {}) + assert query_to_str == "SELECT name FROM users WHERE email = 'test@example.com'" diff --git a/tests/ops/service/connectors/test_snowflake_query_config.py b/tests/ops/service/connectors/test_snowflake_query_config.py index 5521a1a88a..4f4b23b8c4 100644 --- a/tests/ops/service/connectors/test_snowflake_query_config.py +++ b/tests/ops/service/connectors/test_snowflake_query_config.py @@ -150,7 +150,7 @@ def test_generate_update_stmt( ) assert ( str(update_stmt) - == 'UPDATE "address" SET "city" = :city, "house" = :house, "state" = :state, "street" = :street, "zip" = :zip WHERE "id" = :id' + == 'UPDATE "address" SET "city" = :masked_city, "house" = :masked_house, "state" = :masked_state, "street" = :masked_street, "zip" = :masked_zip WHERE "id" = :id' ) def test_generate_namespaced_update_stmt( @@ -191,5 +191,5 @@ def test_generate_namespaced_update_stmt( ) assert ( str(update_stmt) - == 'UPDATE "FIDESOPS_TEST"."TEST"."address" SET "city" = :city, "house" = :house, "state" = :state, "street" = :street, "zip" = :zip WHERE "id" = :id' + == 'UPDATE "FIDESOPS_TEST"."TEST"."address" SET "city" = :masked_city, "house" = :masked_house, "state" = :masked_state, "street" = :masked_street, "zip" = :masked_zip WHERE "id" = :id' ) diff --git a/tests/ops/service/dataset/example_datasets/multiple_identities.yml b/tests/ops/service/dataset/example_datasets/multiple_identities.yml index 053afb3ced..dd76dbfa6d 100644 --- a/tests/ops/service/dataset/example_datasets/multiple_identities.yml +++ b/tests/ops/service/dataset/example_datasets/multiple_identities.yml @@ -16,8 +16,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml b/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml index fdfcd32bfc..db9e227a74 100644 --- a/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml +++ b/tests/ops/service/dataset/example_datasets/multiple_identities_with_external_dependency.yml @@ -32,7 +32,5 @@ dataset: direction: from - name: id data_categories: [system.operations] - fides_meta: - primary_key: True - name: shipping_address_id data_categories: [system.operations] diff --git a/tests/ops/service/dataset/example_datasets/no_identities.yml b/tests/ops/service/dataset/example_datasets/no_identities.yml index fac879de99..82b56f9c65 100644 --- a/tests/ops/service/dataset/example_datasets/no_identities.yml +++ b/tests/ops/service/dataset/example_datasets/no_identities.yml @@ -13,8 +13,6 @@ dataset: data_categories: [user.contact.email] - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/dataset/example_datasets/single_identity.yml b/tests/ops/service/dataset/example_datasets/single_identity.yml index 19cdc7df3e..ce1506886d 100644 --- a/tests/ops/service/dataset/example_datasets/single_identity.yml +++ b/tests/ops/service/dataset/example_datasets/single_identity.yml @@ -16,8 +16,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml b/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml index 708aefbaf0..af73f8bcb8 100644 --- a/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml +++ b/tests/ops/service/dataset/example_datasets/single_identity_with_internal_dependency.yml @@ -16,8 +16,6 @@ dataset: data_type: string - name: id data_categories: [user.unique_id] - fides_meta: - primary_key: True - name: name data_categories: [user.name] fides_meta: diff --git a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py index 8fb7e29729..5a133c031f 100644 --- a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py +++ b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py @@ -1,27 +1,9 @@ -import time -from datetime import datetime, timezone -from typing import Any, Dict, List, Set from unittest import mock -from unittest.mock import ANY, Mock, call -from uuid import uuid4 -import pydash import pytest from fides.api.models.audit_log import AuditLog, AuditLogAction -from fides.api.models.privacy_request import ( - ActionType, - CheckpointActionRequired, - ExecutionLog, - ExecutionLogStatus, - PolicyPreWebhook, - PrivacyRequest, - PrivacyRequestStatus, -) -from fides.api.schemas.masking.masking_configuration import MaskingConfiguration -from fides.api.schemas.masking.masking_secrets import MaskingSecretCache -from fides.api.schemas.policy import Rule -from fides.api.service.masking.strategy.masking_strategy import MaskingStrategy +from fides.api.models.privacy_request import ExecutionLog from tests.ops.service.privacy_request.test_request_runner_service import ( get_privacy_request_results, ) @@ -54,7 +36,7 @@ def test_create_and_process_access_request_bigquery_enterprise( customer_email = "customer-1@example.com" user_id = ( - 1754 # this is a real (not generated) user id in the Stackoverflow dataset + 1754 # this is a real (not generated) user id in the Stack Overflow dataset ) data = { "requested_at": "2024-08-30T16:09:37.359Z", diff --git a/tests/ops/service/privacy_request/test_postgres_privacy_requests.py b/tests/ops/service/privacy_request/test_postgres_privacy_requests.py index 2959efd463..3f5ec661f5 100644 --- a/tests/ops/service/privacy_request/test_postgres_privacy_requests.py +++ b/tests/ops/service/privacy_request/test_postgres_privacy_requests.py @@ -160,9 +160,16 @@ def test_upload_access_results_has_data_use_map( "dsr_version", ["use_dsr_3_0", "use_dsr_2_0"], ) +@pytest.mark.parametrize( + "dataset_config", + [ + "postgres_example_test_dataset_config_read_access", + "postgres_example_test_dataset_config_read_access_without_primary_keys", + ], +) def test_create_and_process_access_request_postgres( trigger_webhook_mock, - postgres_example_test_dataset_config_read_access, + dataset_config, postgres_integration_db, db, cache, @@ -174,6 +181,7 @@ def test_create_and_process_access_request_postgres( run_privacy_request_task, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + request.getfixturevalue(dataset_config) customer_email = "customer-1@example.com" data = { @@ -196,7 +204,7 @@ def test_create_and_process_access_request_postgres( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"postgres_example_test_dataset:" + result_key_prefix = "postgres_example_test_dataset:" customer_key = result_key_prefix + "customer" assert results[customer_key][0]["email"] == customer_email @@ -278,14 +286,14 @@ def test_create_and_process_access_request_with_custom_identities_postgres( assert results[key] is not None assert results[key] != {} - result_key_prefix = f"postgres_example_test_dataset:" + result_key_prefix = "postgres_example_test_dataset:" customer_key = result_key_prefix + "customer" assert results[customer_key][0]["email"] == customer_email visit_key = result_key_prefix + "visit" assert results[visit_key][0]["email"] == customer_email - loyalty_key = f"postgres_example_test_extended_dataset:loyalty" + loyalty_key = "postgres_example_test_extended_dataset:loyalty" assert results[loyalty_key][0]["id"] == loyalty_id log_id = pr.execution_logs[0].id @@ -355,7 +363,7 @@ def test_create_and_process_access_request_with_valid_skipped_collection( assert "login" not in results.keys() - result_key_prefix = f"postgres_example_test_dataset:" + result_key_prefix = "postgres_example_test_dataset:" customer_key = result_key_prefix + "customer" assert results[customer_key][0]["email"] == customer_email @@ -712,9 +720,16 @@ def test_create_and_process_erasure_request_with_table_joins( "dsr_version", ["use_dsr_3_0", "use_dsr_2_0"], ) +@pytest.mark.parametrize( + "dataset_config", + [ + "postgres_example_test_dataset_config_read_access", + "postgres_example_test_dataset_config_read_access_without_primary_keys", + ], +) def test_create_and_process_erasure_request_read_access( postgres_integration_db, - postgres_example_test_dataset_config_read_access, + dataset_config, db, cache, erasure_policy, @@ -723,6 +738,7 @@ def test_create_and_process_erasure_request_read_access( run_privacy_request_task, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + request.getfixturevalue(dataset_config) customer_email = "customer-2@example.com" customer_id = 2 @@ -739,7 +755,7 @@ def test_create_and_process_erasure_request_read_access( data, ) errored_execution_logs = pr.execution_logs.filter_by(status="error") - assert errored_execution_logs.count() == 9 + assert errored_execution_logs.count() == 11 assert ( errored_execution_logs[0].message == "No values were erased since this connection " diff --git a/tests/ops/test_helpers/dataset_utils.py b/tests/ops/test_helpers/dataset_utils.py index e60efb9892..d51e1f47ff 100644 --- a/tests/ops/test_helpers/dataset_utils.py +++ b/tests/ops/test_helpers/dataset_utils.py @@ -13,7 +13,11 @@ ) from fides.api.graph.data_type import DataType, get_data_type, to_data_type_string from fides.api.models.connectionconfig import ConnectionConfig -from fides.api.models.datasetconfig import DatasetConfig, convert_dataset_to_graph +from fides.api.models.datasetconfig import ( + DatasetConfig, + DatasetField, + convert_dataset_to_graph, +) from fides.api.util.collection_util import Row SAAS_DATASET_DIRECTORY = "data/saas/dataset/" @@ -231,3 +235,27 @@ def get_simple_fields(fields: Iterable[Field]) -> List[Dict[str, Any]]: object["fields"] = get_simple_fields(field.fields.values()) object_list.append(object) return object_list + + +def remove_primary_keys(dataset: Dataset) -> Dataset: + """Returns a copy of the dataset with primary key fields removed from fides_meta.""" + dataset_copy = dataset.model_copy(deep=True) + + for collection in dataset_copy.collections: + for field in collection.fields: + if field.fides_meta: + if field.fides_meta.primary_key: + field.fides_meta.primary_key = None + if field.fields: + _remove_nested_primary_keys(field.fields) + + return dataset_copy + + +def _remove_nested_primary_keys(fields: List[DatasetField]) -> None: + """Helper function to recursively remove primary keys from nested fields.""" + for field in fields: + if field.fides_meta and field.fides_meta.primary_key: + field.fides_meta.primary_key = None + if field.fields: + _remove_nested_primary_keys(field.fields) From 4103da1233ec40fa1696451dc2a37032ca6be5b1 Mon Sep 17 00:00:00 2001 From: erosselli <67162025+erosselli@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:10:10 -0300 Subject: [PATCH 11/50] HJ-319 Updated updated_at field of DBCache even when value doesn't change (#5670) --- CHANGELOG.md | 1 + src/fides/api/models/db_cache.py | 4 ++++ tests/ops/models/test_dbcache.py | 13 +++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eb9994d6d..a31b791e82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) - Changed "Reclassify" D&D button to show in an overflow menu when row actions are overcrowded [#5655](https://github.com/ethyca/fides/pull/5655) - Removed primary key requirements for BigQuery and Postgres erasures [#5591](https://github.com/ethyca/fides/pull/5591) +- Updated `DBCache` model so setting cache value always updates the updated_at field [#5669](https://github.com/ethyca/fides/pull/5669) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/src/fides/api/models/db_cache.py b/src/fides/api/models/db_cache.py index 7d19023d4b..ff9c89da81 100644 --- a/src/fides/api/models/db_cache.py +++ b/src/fides/api/models/db_cache.py @@ -4,6 +4,7 @@ from sqlalchemy import Column, Index, String from sqlalchemy.dialects.postgresql import BYTEA from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified from fides.api.db.base_class import Base @@ -78,6 +79,9 @@ def set_cache_value( db_cache_entry = cls.get_cache_entry(db, namespace, cache_key) if db_cache_entry: db_cache_entry.cache_value = cache_value + # We manually flag it as modified so that the update runs even if the cache_value hasn't changed + # so the updated_at field of the cache entry gets updated. + flag_modified(db_cache_entry, "cache_value") else: db_cache_entry = cls( namespace=namespace.value, cache_key=cache_key, cache_value=cache_value diff --git a/tests/ops/models/test_dbcache.py b/tests/ops/models/test_dbcache.py index 7fe9062fab..f20e68df86 100644 --- a/tests/ops/models/test_dbcache.py +++ b/tests/ops/models/test_dbcache.py @@ -47,12 +47,16 @@ def test_update_cache_value(self, db): ).decode() == "value 1" ) + assert cache_value.updated_at is not None + + original_timestamp = cache_value.updated_at # Update the cache value cache_value = DBCache.set_cache_value( db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key", "value 2".encode() ) assert cache_value.cache_value.decode() == "value 2" + assert cache_value.updated_at > original_timestamp # Check the value was actually updated updated_value = DBCache.get_cache_value( @@ -60,6 +64,15 @@ def test_update_cache_value(self, db): ) assert updated_value.decode() == "value 2" + previous_timestamp = cache_value.updated_at + + # Updating the value with the same value should still update the timestamp + cache_value = DBCache.set_cache_value( + db, DBCacheNamespace.LIST_PRIVACY_EXPERIENCE, "some-key", "value 2".encode() + ) + assert cache_value.cache_value.decode() == "value 2" + assert cache_value.updated_at > previous_timestamp + def test_delete_cache_entry(self, db): # Add two entries DBCache.set_cache_value( From 31dcf581537828acfc6cb9628277628572bcbe04 Mon Sep 17 00:00:00 2001 From: Catherine Smith Date: Wed, 15 Jan 2025 12:34:42 +0100 Subject: [PATCH 12/50] LA-108: Expand on BQ Enterprise test coverage- partitioning, custom identities (#5618) --- CHANGELOG.md | 1 + .../bigquery_enterprise_test_dataset.yml | 55 ++++ tests/fixtures/bigquery_fixtures.py | 177 ++++++++++- ...est_bigquery_enterprise_privacy_request.py | 287 +++++++++++++++++- 4 files changed, 505 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a31b791e82..75fa9987f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o ### Added - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) - Added cache-clearing methods to the `DBCache` model to allow deleting cache entries [#5629](https://github.com/ethyca/fides/pull/5629) +- Adds partitioning, custom identities, multiple identities to test coverage for BigQuery Enterprise [#5618](https://github.com/ethyca/fides/pull/5618) ### Changed - Updated brand link url [#5656](https://github.com/ethyca/fides/pull/5656) diff --git a/data/dataset/bigquery_enterprise_test_dataset.yml b/data/dataset/bigquery_enterprise_test_dataset.yml index 64668192d0..f2532009cb 100644 --- a/data/dataset/bigquery_enterprise_test_dataset.yml +++ b/data/dataset/bigquery_enterprise_test_dataset.yml @@ -112,6 +112,61 @@ dataset: data_categories: [user.contact] - name: view_count data_categories: [system.operations] + - name: stackoverflow_posts_partitioned + fields: + - name: accepted_answer_id + data_categories: [ system.operations ] + - name: answer_count + data_categories: [ system.operations ] + - name: body + data_categories: [ user.contact ] + - name: comment_count + data_categories: [ system.operations ] + - name: community_owned_date + data_categories: [ system.operations ] + - name: creation_date + data_categories: [ system.operations ] + - name: favorite_count + data_categories: [ system.operations ] + - name: id + data_categories: [ system.operations ] + fides_meta: + data_type: integer + - name: last_activity_date + data_categories: [ system.operations ] + - name: last_edit_date + data_categories: [ system.operations ] + - name: last_editor_display_name + data_categories: [ system.operations ] + - name: last_editor_user_id + data_categories: [ system.operations ] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + - name: owner_display_name + data_categories: [ user.contact ] + - name: owner_user_id + data_categories: [ system.operations ] + fides_meta: + references: + - dataset: enterprise_dsr_testing + field: users.id + direction: from + data_type: integer + - name: parent_id + data_categories: [ system.operations ] + - name: post_type_id + data_categories: [ system.operations ] + - name: score + data_categories: [ system.operations ] + - name: tags + data_categories: [ system.operations ] + - name: title + data_categories: [ user.contact ] + - name: view_count + data_categories: [ system.operations] - name: users fields: - name: about_me diff --git a/tests/fixtures/bigquery_fixtures.py b/tests/fixtures/bigquery_fixtures.py index 105910e466..982e18a12b 100644 --- a/tests/fixtures/bigquery_fixtures.py +++ b/tests/fixtures/bigquery_fixtures.py @@ -202,6 +202,56 @@ def bigquery_enterprise_test_dataset_config( ctl_dataset.delete(db=db) +@pytest.fixture +def bigquery_enterprise_test_dataset_config_with_partitioning_meta( + bigquery_enterprise_connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], +) -> Generator: + bigquery_enterprise_dataset = example_datasets[16] + fides_key = bigquery_enterprise_dataset["fides_key"] + bigquery_enterprise_connection_config.name = fides_key + bigquery_enterprise_connection_config.key = fides_key + + # Update stackoverflow_posts_partitioned collection to have partition meta_data + # It is already set up as a partitioned table in BigQuery itself + stackoverflow_posts_partitioned_collection = next( + collection + for collection in bigquery_enterprise_dataset["collections"] + if collection["name"] == "stackoverflow_posts_partitioned" + ) + bigquery_enterprise_dataset["collections"].remove( + stackoverflow_posts_partitioned_collection + ) + stackoverflow_posts_partitioned_collection["fides_meta"] = { + "partitioning": { + "where_clauses": [ + "`creation_date` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY) AND `creation_date` <= CURRENT_TIMESTAMP()", + "`creation_date` > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 2000 DAY) AND `creation_date` <= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1000 DAY)", + ] + } + } + bigquery_enterprise_dataset["collections"].append( + stackoverflow_posts_partitioned_collection + ) + + bigquery_enterprise_connection_config.save(db=db) + + ctl_dataset = CtlDataset.create_from_dataset_dict(db, bigquery_enterprise_dataset) + + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": bigquery_enterprise_connection_config.id, + "fides_key": fides_key, + "ctl_dataset_id": ctl_dataset.id, + }, + ) + yield dataset + dataset.delete(db=db) + ctl_dataset.delete(db=db) + + @pytest.fixture def bigquery_example_test_dataset_config_with_namespace_meta( bigquery_connection_config_without_default_dataset: ConnectionConfig, @@ -482,14 +532,14 @@ def bigquery_enterprise_resources( """ connection.execute(stmt) - # Create test stackoverflow_posts data. Posts are responses to questions on Stackoverflow, and does not include original question. + # Create test stackoverflow_posts_partitioned data. Posts are responses to questions on Stackoverflow, and does not include original question. post_body = "For me, the solution was to adopt 3 cats and dance with them under the full moon at midnight." - stmt = "select max(id) from enterprise_dsr_testing.stackoverflow_posts;" + stmt = "select max(id) from enterprise_dsr_testing.stackoverflow_posts_partitioned;" res = connection.execute(stmt) random_increment = random.randint(0, 99999) post_id = res.all()[0][0] + random_increment stmt = f""" - insert into enterprise_dsr_testing.stackoverflow_posts (body, creation_date, id, owner_user_id, owner_display_name) + insert into enterprise_dsr_testing.stackoverflow_posts_partitioned (body, creation_date, id, owner_user_id, owner_display_name) values ('{post_body}', '{creation_date}', {post_id}, {user_id}, '{display_name}'); """ connection.execute(stmt) @@ -539,7 +589,102 @@ def bigquery_enterprise_resources( stmt = f"delete from enterprise_dsr_testing.comments where id = {comment_id};" connection.execute(stmt) - stmt = f"delete from enterprise_dsr_testing.stackoverflow_posts where id = {post_id};" + stmt = f"delete from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" + connection.execute(stmt) + + stmt = f"delete from enterprise_dsr_testing.users where id = {user_id};" + connection.execute(stmt) + + +@pytest.fixture(scope="function") +def bigquery_enterprise_resources_with_partitioning( + bigquery_enterprise_test_dataset_config_with_partitioning_meta, +): + bigquery_connection_config = ( + bigquery_enterprise_test_dataset_config_with_partitioning_meta.connection_config + ) + connector = BigQueryConnector(bigquery_connection_config) + bigquery_client = connector.client() + with bigquery_client.connect() as connection: + + # Real max id in the Stackoverflow dataset is 20081052, so we purposefully generate and id above this max + stmt = "select max(id) from enterprise_dsr_testing.users;" + res = connection.execute(stmt) + # Increment the id by a random number to avoid conflicts on concurrent test runs + random_increment = random.randint(0, 99999) + user_id = res.all()[0][0] + random_increment + display_name = ( + f"fides_testing_{user_id}" # prefix to do manual cleanup if needed + ) + last_access_date = datetime.now() + creation_date = datetime.now() + location = "Dream World" + + # Create test user data + stmt = f""" + insert into enterprise_dsr_testing.users (id, display_name, last_access_date, creation_date, location) + values ({user_id}, '{display_name}', '{last_access_date}', '{creation_date}', '{location}'); + """ + connection.execute(stmt) + + # Create test stackoverflow_posts_partitioned data. Posts are responses to questions on Stackoverflow, and does not include original question. + post_body = "For me, the solution was to adopt 3 cats and dance with them under the full moon at midnight." + stmt = "select max(id) from enterprise_dsr_testing.stackoverflow_posts_partitioned;" + res = connection.execute(stmt) + random_increment = random.randint(0, 99999) + post_id = res.all()[0][0] + random_increment + stmt = f""" + insert into enterprise_dsr_testing.stackoverflow_posts_partitioned (body, creation_date, id, owner_user_id, owner_display_name) + values ('{post_body}', '{creation_date}', {post_id}, {user_id}, '{display_name}'); + """ + connection.execute(stmt) + + # Create test comments data. Comments are responses to posts or questions on Stackoverflow, and does not include original question or post itself. + stmt = "select max(id) from enterprise_dsr_testing.comments;" + res = connection.execute(stmt) + random_increment = random.randint(0, 99999) + comment_id = res.all()[0][0] + random_increment + comment_text = "FYI this only works if you have pytest installed locally." + stmt = f""" + insert into enterprise_dsr_testing.comments (id, text, creation_date, post_id, user_id, user_display_name) + values ({comment_id}, '{comment_text}', '{creation_date}', {post_id}, {user_id}, '{display_name}'); + """ + connection.execute(stmt) + + # Create test post_history data + stmt = "select max(id) from enterprise_dsr_testing.comments;" + res = connection.execute(stmt) + random_increment = random.randint(0, 99999) + post_history_id = res.all()[0][0] + random_increment + revision_text = "this works if you have pytest" + uuid = str(uuid4()) + stmt = f""" + insert into enterprise_dsr_testing.post_history (id, text, creation_date, post_id, user_id, post_history_type_id, revision_guid) + values ({post_history_id}, '{revision_text}', '{creation_date}', {post_id}, {user_id}, 1, '{uuid}'); + """ + connection.execute(stmt) + + yield { + "name": display_name, + "user_id": user_id, + "comment_id": comment_id, + "post_history_id": post_history_id, + "post_id": post_id, + "client": bigquery_client, + "connector": connector, + "first_comment_text": comment_text, + "first_post_body": post_body, + "revision_text": revision_text, + "display_name": display_name, + } + # Remove test data and close BigQuery connection in teardown + stmt = f"delete from enterprise_dsr_testing.post_history where id = {post_history_id};" + connection.execute(stmt) + + stmt = f"delete from enterprise_dsr_testing.comments where id = {comment_id};" + connection.execute(stmt) + + stmt = f"delete from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" connection.execute(stmt) stmt = f"delete from enterprise_dsr_testing.users where id = {user_id};" @@ -571,6 +716,30 @@ def bigquery_test_engine(bigquery_keyfile_creds) -> Generator: engine.dispose() +def seed_bigquery_enterprise_integration_db( + bigquery_enterprise_test_dataset_config, +) -> None: + """ + Currently unused. + This helper function has already been run once, and data has been populated in the test BigQuery enterprise dataset. + We may need this later in case tables are accidentally removed. + """ + bigquery_connection_config = ( + bigquery_enterprise_test_dataset_config.connection_config + ) + connector = BigQueryConnector(bigquery_connection_config) + bigquery_client = connector.client() + with bigquery_client.connect() as connection: + + stmt = f"CREATE TABLE enterprise_dsr_testing.stackoverflow_posts_partitioned partition by date(creation_date) as select * from enterprise_dsr_testing.stackoverflow_posts;" + connection.execute(stmt) + + print( + f"Created table enterprise_dsr_testing.stackoverflow_posts_partitioned, " + f"partitioned on column creation_date." + ) + + def seed_bigquery_integration_db(bigquery_integration_engine) -> None: """ Currently unused. diff --git a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py index 5a133c031f..9042d4758a 100644 --- a/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py +++ b/tests/ops/service/privacy_request/test_bigquery_enterprise_privacy_request.py @@ -10,29 +10,38 @@ PRIVACY_REQUEST_TASK_TIMEOUT = 5 # External services take much longer to return -PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL = 100 +PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL = 150 @pytest.mark.integration_bigquery @pytest.mark.integration_external @pytest.mark.parametrize( "dsr_version", - ["use_dsr_3_0", "use_dsr_2_0"], + ["use_dsr_2_0", "use_dsr_3_0"], +) +@pytest.mark.parametrize( + "bigquery_fixtures", + [ + "bigquery_enterprise_test_dataset_config", + ], ) @mock.patch("fides.api.models.privacy_request.PrivacyRequest.trigger_policy_webhook") -def test_create_and_process_access_request_bigquery_enterprise( +def test_access_request( trigger_webhook_mock, - bigquery_enterprise_test_dataset_config, db, cache, policy, dsr_version, + bigquery_fixtures, request, policy_pre_execution_webhooks, policy_post_execution_webhooks, run_privacy_request_task, ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + request.getfixturevalue( + bigquery_fixtures + ) # required to test partitioning and non-partitioned tables customer_email = "customer-1@example.com" user_id = ( @@ -89,7 +98,9 @@ def test_create_and_process_access_request_bigquery_enterprise( len( [ post["title"] - for post in results["enterprise_dsr_testing:stackoverflow_posts"] + for post in results[ + "enterprise_dsr_testing:stackoverflow_posts_partitioned" + ] ] ) == 30 @@ -132,17 +143,17 @@ def test_create_and_process_access_request_bigquery_enterprise( @pytest.mark.parametrize( "bigquery_fixtures", [ - "bigquery_enterprise_resources" - ], # todo- add other resources to test, e.g. partitioned data + "bigquery_enterprise_resources", + "bigquery_enterprise_resources_with_partitioning", + ], ) -def test_create_and_process_erasure_request_bigquery( +def test_erasure_request( db, request, policy, cache, dsr_version, bigquery_fixtures, - bigquery_enterprise_test_dataset_config, bigquery_enterprise_erasure_policy, run_privacy_request_task, ): @@ -204,7 +215,9 @@ def test_create_and_process_erasure_request_bigquery( len( [ post["title"] - for post in results["enterprise_dsr_testing:stackoverflow_posts"] + for post in results[ + "enterprise_dsr_testing:stackoverflow_posts_partitioned" + ] ] ) == 1 @@ -248,7 +261,259 @@ def test_create_and_process_erasure_request_bigquery( assert row.user_display_name is None assert row.text is None - stmt = f"select owner_user_id, owner_display_name, body from enterprise_dsr_testing.stackoverflow_posts where id = {post_id};" + stmt = f"select owner_user_id, owner_display_name, body from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" + res = connection.execute(stmt).all() + for row in res: + assert ( + row.owner_user_id == bigquery_enterprise_resources["user_id"] + ) # not targeted by policy + assert row.owner_display_name is None + assert row.body is None + + stmt = f"select display_name, location from enterprise_dsr_testing.users where id = {user_id};" + res = connection.execute(stmt).all() + for row in res: + assert row.display_name is None + assert row.location is None + + +@pytest.mark.integration_bigquery +@pytest.mark.integration_external +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0"], +) +@mock.patch("fides.api.models.privacy_request.PrivacyRequest.trigger_policy_webhook") +def test_access_request_multiple_custom_identities( + trigger_webhook_mock, + bigquery_enterprise_test_dataset_config, + db, + cache, + policy, + dsr_version, + request, + policy_pre_execution_webhooks, + policy_post_execution_webhooks, + run_privacy_request_task, +): + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + + user_id = ( + 1754 # this is a real (not generated) user id in the Stackoverflow dataset + ) + data = { + "requested_at": "2024-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": { + "loyalty_id": {"label": "Loyalty ID", "value": "CH-1"}, + "stackoverflow_user_id": { + "label": "Stackoverflow User Id", + "value": user_id, + }, + }, + } + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + data, + PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, + ) + + results = pr.get_raw_access_results() + assert len(results.keys()) == 4 + + for key in results.keys(): + assert results[key] is not None + assert results[key] != {} + + users = results["enterprise_dsr_testing:users"] + assert len(users) == 1 + user_details = users[0] + assert user_details["id"] == user_id + + assert ( + len( + [ + comment["user_id"] + for comment in results["enterprise_dsr_testing:comments"] + ] + ) + == 16 + ) + assert ( + len( + [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] + ) + == 39 + ) + assert ( + len( + [ + post["title"] + for post in results[ + "enterprise_dsr_testing:stackoverflow_posts_partitioned" + ] + ] + ) + == 30 + ) + + log_id = pr.execution_logs[0].id + pr_id = pr.id + + finished_audit_log: AuditLog = AuditLog.filter( + db=db, + conditions=( + (AuditLog.privacy_request_id == pr_id) + & (AuditLog.action == AuditLogAction.finished) + ), + ).first() + + assert finished_audit_log is not None + + # Both pre-execution webhooks and both post-execution webhooks were called + assert trigger_webhook_mock.call_count == 4 + + for webhook in policy_pre_execution_webhooks: + webhook.delete(db=db) + + for webhook in policy_post_execution_webhooks: + webhook.delete(db=db) + + policy.delete(db=db) + pr.delete(db=db) + assert not pr in db # Check that `pr` has been expunged from the session + assert ExecutionLog.get(db, object_id=log_id).privacy_request_id == pr_id + + +@pytest.mark.integration_external +@pytest.mark.integration_bigquery +@pytest.mark.parametrize( + "dsr_version", + ["use_dsr_3_0"], +) +@pytest.mark.parametrize( + "bigquery_fixtures", + [ + "bigquery_enterprise_resources", + "bigquery_enterprise_resources_with_partitioning", + ], +) +def test_erasure_request_multiple_custom_identities( + db, + request, + policy, + cache, + dsr_version, + bigquery_fixtures, + bigquery_enterprise_erasure_policy, + run_privacy_request_task, +): + request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 + bigquery_enterprise_resources = request.getfixturevalue(bigquery_fixtures) + bigquery_client = bigquery_enterprise_resources["client"] + + # first test access request against manually added data + user_id = bigquery_enterprise_resources["user_id"] + data = { + "requested_at": "2024-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": { + "loyalty_id": {"label": "Loyalty ID", "value": "CH-1"}, + "stackoverflow_user_id": { + "label": "Stackoverflow User Id", + "value": user_id, + }, + }, + } + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + data, + PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, + ) + + results = pr.get_raw_access_results() + assert len(results.keys()) == 4 + + for key in results.keys(): + assert results[key] is not None + assert results[key] != {} + + users = results["enterprise_dsr_testing:users"] + assert len(users) == 1 + user_details = users[0] + assert user_details["id"] == user_id + + assert ( + len( + [ + comment["user_id"] + for comment in results["enterprise_dsr_testing:comments"] + ] + ) + == 1 + ) + assert ( + len( + [post["user_id"] for post in results["enterprise_dsr_testing:post_history"]] + ) + == 1 + ) + assert ( + len( + [ + post["title"] + for post in results[ + "enterprise_dsr_testing:stackoverflow_posts_partitioned" + ] + ] + ) + == 1 + ) + + data = { + "requested_at": "2024-08-30T16:09:37.359Z", + "policy_key": bigquery_enterprise_erasure_policy.key, + "identity": { + "stackoverflow_user_id": { + "label": "Stackoverflow User Id", + "value": bigquery_enterprise_resources["user_id"], + }, + }, + } + + # Should erase all user data + pr = get_privacy_request_results( + db, + bigquery_enterprise_erasure_policy, + run_privacy_request_task, + data, + task_timeout=PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, + ) + pr.delete(db=db) + + bigquery_client = bigquery_enterprise_resources["client"] + post_history_id = bigquery_enterprise_resources["post_history_id"] + comment_id = bigquery_enterprise_resources["comment_id"] + post_id = bigquery_enterprise_resources["post_id"] + with bigquery_client.connect() as connection: + stmt = f"select text from enterprise_dsr_testing.post_history where id = {post_history_id};" + res = connection.execute(stmt).all() + for row in res: + assert row.text is None + + stmt = f"select user_display_name, text from enterprise_dsr_testing.comments where id = {comment_id};" + res = connection.execute(stmt).all() + for row in res: + assert row.user_display_name is None + assert row.text is None + + stmt = f"select owner_user_id, owner_display_name, body from enterprise_dsr_testing.stackoverflow_posts_partitioned where id = {post_id};" res = connection.execute(stmt).all() for row in res: assert ( From a4f4aec1a4b40ec8dbfee71d89dcef97554b7b09 Mon Sep 17 00:00:00 2001 From: jpople Date: Wed, 15 Jan 2025 11:33:48 -0600 Subject: [PATCH 13/50] Adjust button sizes in table headers (#5654) --- CHANGELOG.md | 1 + .../common/custom-reports/CustomReportTemplates.tsx | 1 - .../features/configure-consent/ConsentManagementTable.tsx | 4 +--- .../features/custom-assets/CustomAssetUploadButton.tsx | 8 +------- .../src/features/custom-fields/CustomFieldsTable.tsx | 1 - .../tables/DiscoveryTableBulkActions.tsx | 2 ++ .../src/features/datamap/reporting/DatamapReportTable.tsx | 4 ---- .../integrations/configure-monitor/MonitorConfigTab.tsx | 1 - .../src/features/privacy-experience/JavaScriptTag.tsx | 1 - .../privacy-experience/PrivacyExperiencesTable.tsx | 3 +-- .../src/features/privacy-notices/PrivacyNoticesTable.tsx | 6 +----- .../src/features/privacy-requests/RequestTable.tsx | 5 ++--- .../features/privacy-requests/SubmitPrivacyRequest.tsx | 7 +------ .../features/privacy-requests/drawers/ConfigureAlerts.tsx | 1 - .../admin-ui/src/features/properties/PropertiesTable.tsx | 5 +---- .../system/add-multiple-systems/AddMultipleSystems.tsx | 6 +----- clients/admin-ui/src/pages/messaging/index.tsx | 1 - 17 files changed, 12 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75fa9987f6..0497ab0779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Changed "Reclassify" D&D button to show in an overflow menu when row actions are overcrowded [#5655](https://github.com/ethyca/fides/pull/5655) - Removed primary key requirements for BigQuery and Postgres erasures [#5591](https://github.com/ethyca/fides/pull/5591) - Updated `DBCache` model so setting cache value always updates the updated_at field [#5669](https://github.com/ethyca/fides/pull/5669) +- Changed sizes of buttons in table headers [#5654](https://github.com/ethyca/fides/pull/5654) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/admin-ui/src/features/common/custom-reports/CustomReportTemplates.tsx b/clients/admin-ui/src/features/common/custom-reports/CustomReportTemplates.tsx index ffcd22dc18..94e8c3e37d 100644 --- a/clients/admin-ui/src/features/common/custom-reports/CustomReportTemplates.tsx +++ b/clients/admin-ui/src/features/common/custom-reports/CustomReportTemplates.tsx @@ -230,7 +230,6 @@ export const CustomReportTemplates = ({ > diff --git a/clients/admin-ui/src/features/custom-assets/CustomAssetUploadButton.tsx b/clients/admin-ui/src/features/custom-assets/CustomAssetUploadButton.tsx index 0f41b5f830..839a0a0c13 100644 --- a/clients/admin-ui/src/features/custom-assets/CustomAssetUploadButton.tsx +++ b/clients/admin-ui/src/features/custom-assets/CustomAssetUploadButton.tsx @@ -16,13 +16,7 @@ const CustomAssetUploadButton = ({ return ( <> - + { const AddCustomFieldButton = () => (
diff --git a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx index 7960003dda..f9843a1c06 100644 --- a/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx +++ b/clients/admin-ui/src/features/datamap/reporting/DatamapReportTable.tsx @@ -527,7 +527,6 @@ export const DatamapReportTable = () => { } iconPosition="end" loading={groupChangeStarted} @@ -560,7 +559,6 @@ export const DatamapReportTable = () => { diff --git a/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx b/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx index 062a6c5964..7d33905dd6 100644 --- a/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx +++ b/clients/admin-ui/src/features/privacy-requests/RequestTable.tsx @@ -139,14 +139,13 @@ export const RequestTable = ({ ...props }: BoxProps): JSX.Element => { setGlobalFilter={handleSearch} placeholder="Search by request ID or identity value" /> - - diff --git a/clients/admin-ui/src/features/privacy-requests/drawers/ConfigureAlerts.tsx b/clients/admin-ui/src/features/privacy-requests/drawers/ConfigureAlerts.tsx index 2f7b49b0de..144fedbcdd 100644 --- a/clients/admin-ui/src/features/privacy-requests/drawers/ConfigureAlerts.tsx +++ b/clients/admin-ui/src/features/privacy-requests/drawers/ConfigureAlerts.tsx @@ -111,7 +111,6 @@ const ConfigureAlerts = () => { <> - } - /> - ))} + {filteredTypes.map((i) => { + if (!datahub && i.placeholder.connection_type === "datahub") { + return null; + } + return ( + onConfigureClick(i)} + otherButtons={ + + } + /> + ); + })} )} diff --git a/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx b/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx index c2211950d9..468110fab3 100644 --- a/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx +++ b/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx @@ -2,6 +2,7 @@ import { ReactNode } from "react"; import { ConnectionCategory } from "~/features/integrations/ConnectionCategory"; import BIGQUERY_TYPE_INFO from "~/features/integrations/integration-type-info/bigqueryInfo"; +import DATAHUB_TYPE_INFO from "~/features/integrations/integration-type-info/datahubInfo"; import DYNAMO_TYPE_INFO from "~/features/integrations/integration-type-info/dynamoInfo"; import GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLMySQLInfo"; import GOOGLE_CLOUD_SQL_POSTGRES_TYPE_INFO from "~/features/integrations/integration-type-info/googleCloudSQLPostgresInfo"; @@ -27,6 +28,7 @@ export type IntegrationTypeInfo = { const INTEGRATION_TYPE_MAP: { [K in ConnectionType]?: IntegrationTypeInfo } = { [ConnectionType.BIGQUERY]: BIGQUERY_TYPE_INFO, + [ConnectionType.DATAHUB]: DATAHUB_TYPE_INFO, [ConnectionType.DYNAMODB]: DYNAMO_TYPE_INFO, [ConnectionType.GOOGLE_CLOUD_SQL_MYSQL]: GOOGLE_CLOUD_SQL_MYSQL_TYPE_INFO, [ConnectionType.GOOGLE_CLOUD_SQL_POSTGRES]: diff --git a/clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx b/clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx new file mode 100644 index 0000000000..008af51432 --- /dev/null +++ b/clients/admin-ui/src/features/integrations/integration-type-info/datahubInfo.tsx @@ -0,0 +1,52 @@ +import { ListItem } from "fidesui"; + +import { + InfoHeading, + InfoText, + InfoUnorderedList, +} from "~/features/common/copy/components"; +import ShowMoreContent from "~/features/common/copy/ShowMoreContent"; +import { ConnectionCategory } from "~/features/integrations/ConnectionCategory"; +import { AccessLevel, ConnectionType } from "~/types/api"; + +export const DATAHUB_PLACEHOLDER = { + name: "Datahub", + key: "datahub_placeholder", + connection_type: ConnectionType.DATAHUB, + access: AccessLevel.READ, + created_at: "", +}; + +export const DATAHUB_TAGS = ["Data catalog"]; + +export const DatahubOverview = () => ( + <> + + + DataHub is a metadata platform designed to help organizations manage and + govern their data. It acts as a centralized repository for tracking and + discovering data assets across an organization, helping data teams + understand where their data resides, how it's used, and how it flows + through various systems. + + + + + Data Catalog + + + + Placeholder + + + +); + +const DATAHUB_TYPE_INFO = { + placeholder: DATAHUB_PLACEHOLDER, + category: ConnectionCategory.DATA_CATALOG, + overview: , + tags: DATAHUB_TAGS, +}; + +export default DATAHUB_TYPE_INFO; diff --git a/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx b/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx index 9ed403fb5c..2cfe7c2493 100644 --- a/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx +++ b/clients/admin-ui/src/features/integrations/useIntegrationFilterTabs.tsx @@ -5,6 +5,7 @@ import { IntegrationTypeInfo } from "~/features/integrations/add-integration/all export enum IntegrationFilterTabs { ALL = "All", DATABASE = "Database", + DATA_CATALOG = "Data Catalog", DATA_WAREHOUSE = "Data Warehouse", } diff --git a/clients/admin-ui/src/flags.json b/clients/admin-ui/src/flags.json index 5f2b32cea2..1af54fd494 100644 --- a/clients/admin-ui/src/flags.json +++ b/clients/admin-ui/src/flags.json @@ -53,5 +53,11 @@ "development": true, "test": true, "production": false + }, + "datahub": { + "description": "Share Fides data categories with your Datahub instance", + "development": true, + "test": true, + "production": false } } diff --git a/clients/admin-ui/src/types/api/models/SystemType.ts b/clients/admin-ui/src/types/api/models/SystemType.ts index b08e9d168e..3d1938b963 100644 --- a/clients/admin-ui/src/types/api/models/SystemType.ts +++ b/clients/admin-ui/src/types/api/models/SystemType.ts @@ -3,8 +3,9 @@ /* eslint-disable */ export enum SystemType { - SAAS = "saas", + DATA_CATALOG = "data_catalog", DATABASE = "database", - MANUAL = "manual", EMAIL = "email", + MANUAL = "manual", + SAAS = "saas", } diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 2dc518cd44..6775396734 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -2,7 +2,7 @@ import enum from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type +from typing import TYPE_CHECKING, Any, List, Optional, Type from loguru import logger from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, String, event @@ -24,6 +24,7 @@ if TYPE_CHECKING: from fides.api.models.detection_discovery import MonitorConfig + from fides.api.schemas.connection_configuration.enums.system_type import SystemType class ConnectionTestStatus(enum.Enum): @@ -72,7 +73,7 @@ def human_readable(self) -> str: """Human-readable mapping for ConnectionTypes Add to this mapping if you add a new ConnectionType """ - readable_mapping: Dict[str, str] = { + readable_mapping: dict[str, str] = { ConnectionType.attentive_email.value: "Attentive Email", ConnectionType.bigquery.value: "BigQuery", ConnectionType.datahub.value: "DataHub", @@ -108,6 +109,47 @@ def human_readable(self) -> str: "Add new ConnectionType to human_readable mapping" ) + @property + def system_type(self) -> "SystemType": + from fides.api.schemas.connection_configuration.enums.system_type import ( + SystemType, + ) + + system_type_mapping: dict[str, SystemType] = { + ConnectionType.attentive_email.value: SystemType.email, + ConnectionType.bigquery.value: SystemType.database, + ConnectionType.datahub.value: SystemType.data_catalog, + ConnectionType.dynamic_erasure_email.value: SystemType.email, + ConnectionType.dynamodb.value: SystemType.database, + ConnectionType.fides.value: SystemType.manual, + ConnectionType.generic_consent_email.value: SystemType.email, + ConnectionType.generic_erasure_email.value: SystemType.email, + ConnectionType.google_cloud_sql_mysql.value: SystemType.database, + ConnectionType.google_cloud_sql_postgres.value: SystemType.database, + ConnectionType.https.value: SystemType.manual, + ConnectionType.manual_webhook.value: SystemType.manual, + ConnectionType.manual.value: SystemType.manual, + ConnectionType.mariadb.value: SystemType.database, + ConnectionType.mongodb.value: SystemType.database, + ConnectionType.mssql.value: SystemType.database, + ConnectionType.mysql.value: SystemType.database, + ConnectionType.postgres.value: SystemType.database, + ConnectionType.rds_mysql.value: SystemType.database, + ConnectionType.rds_postgres.value: SystemType.database, + ConnectionType.redshift.value: SystemType.database, + ConnectionType.s3.value: SystemType.database, + ConnectionType.saas.value: SystemType.saas, + ConnectionType.scylla.value: SystemType.database, + ConnectionType.snowflake.value: SystemType.database, + ConnectionType.sovrn.value: SystemType.email, + ConnectionType.timescale.value: SystemType.database, + } + + try: + return system_type_mapping[self.value] + except KeyError: + raise NotImplementedError("Add new ConnectionType to system_type mapping") + class AccessLevel(enum.Enum): """ diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py index bfda71ed50..bde79ad24f 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_datahub.py @@ -1,4 +1,3 @@ -from enum import Enum from typing import ClassVar, List from pydantic import Field @@ -10,14 +9,6 @@ ) -class PeriodicIntegrationFrequency(Enum): - """Enum for periodic integration frequency""" - - daily = "daily" - weekly = "weekly" - monthly = "monthly" - - class DatahubSchema(ConnectionConfigSecretsSchema): datahub_server_url: AnyHttpUrlStringRemovesSlash = Field( title="DataHub Server URL", @@ -28,11 +19,6 @@ class DatahubSchema(ConnectionConfigSecretsSchema): description="The token used to authenticate with your DataHub server.", json_schema_extra={"sensitive": True}, ) - frequency: PeriodicIntegrationFrequency = Field( - title="Frequency", - description="The frequency at which the integration should run. Defaults to daily.", - default=PeriodicIntegrationFrequency.daily, - ) _required_components: ClassVar[List[str]] = ["datahub_server_url", "datahub_token"] diff --git a/src/fides/api/schemas/connection_configuration/enums/system_type.py b/src/fides/api/schemas/connection_configuration/enums/system_type.py index 275a63f3f5..dad517822e 100644 --- a/src/fides/api/schemas/connection_configuration/enums/system_type.py +++ b/src/fides/api/schemas/connection_configuration/enums/system_type.py @@ -2,7 +2,8 @@ class SystemType(Enum): - saas = "saas" + data_catalog = "data_catalog" database = "database" - manual = "manual" email = "email" + manual = "manual" + saas = "saas" diff --git a/src/fides/api/util/connection_type.py b/src/fides/api/util/connection_type.py index 5edae3135a..14c2041d12 100644 --- a/src/fides/api/util/connection_type.py +++ b/src/fides/api/util/connection_type.py @@ -220,9 +220,9 @@ def saas_request_type_filter(connection_type: str) -> bool: if (system_type == SystemType.database or system_type is None) and ( ActionType.access in action_types or ActionType.erasure in action_types ): - database_types: list[str] = sorted( + database_types: list[ConnectionType] = sorted( [ - conn_type.value + conn_type for conn_type in ConnectionType if conn_type not in [ @@ -238,14 +238,15 @@ def saas_request_type_filter(connection_type: str) -> bool: ConnectionType.sovrn, ] and is_match(conn_type.value) - ] + ], + key=lambda x: x.value, ) connection_system_types.extend( [ ConnectionSystemTypeMap( identifier=item, - type=SystemType.database, - human_readable=ConnectionType(item).human_readable, + type=item.system_type, + human_readable=item.human_readable, supported_actions=[ActionType.access, ActionType.erasure], ) for item in database_types @@ -336,4 +337,5 @@ def saas_request_type_filter(connection_type: str) -> bool: for email_type in email_types ] ) + return connection_system_types diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 83fade38c0..3bffd68796 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -2018,7 +2018,6 @@ def test_put_datahub_connection_config_secrets( payload = { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", - "frequency": "weekly", } resp = api_client.put( url + "?verify=False", @@ -2035,7 +2034,6 @@ def test_put_datahub_connection_config_secrets( assert datahub_connection_config_no_secrets.secrets == { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", - "frequency": "weekly", } assert datahub_connection_config_no_secrets.last_test_timestamp is None assert datahub_connection_config_no_secrets.last_test_succeeded is None @@ -2071,7 +2069,6 @@ def test_put_datahub_connection_config_secrets_default_frequency( assert datahub_connection_config_no_secrets.secrets == { "datahub_server_url": "https://datahub.example.com", "datahub_token": "test", - "frequency": "daily", } assert datahub_connection_config_no_secrets.last_test_timestamp is None assert datahub_connection_config_no_secrets.last_test_succeeded is None @@ -2088,7 +2085,7 @@ def test_put_datahub_connection_config_secrets_missing_url( """ url = f"{V1_URL_PREFIX}{CONNECTIONS}/{datahub_connection_config_no_secrets.key}/secret" auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - payload = {"datahub_token": "test", "frequency": "weekly"} + payload = {"datahub_token": "test"} resp = api_client.put( url + "?verify=False", headers=auth_header, @@ -2114,7 +2111,6 @@ def test_put_datahub_connection_config_secrets_missing_token( auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) payload = { "datahub_server_url": "https://datahub.example.com", - "frequency": "weekly", } resp = api_client.put( url + "?verify=False", diff --git a/tests/ops/models/test_connectionconfig.py b/tests/ops/models/test_connectionconfig.py index 93978ab8fc..b24a43dbf3 100644 --- a/tests/ops/models/test_connectionconfig.py +++ b/tests/ops/models/test_connectionconfig.py @@ -146,8 +146,16 @@ def test_connection_type_human_readable(self): connection.human_readable # Makes sure all ConnectionTypes have been added to human_readable mapping def test_connection_type_human_readable_invalid(self): - with pytest.raises(ValueError): - ConnectionType("nonmapped_type").human_readable() + with pytest.raises(NotImplementedError): + ConnectionType("nonmapped_type").human_readable + + def test_connection_type_system_type(self): + for connection in ConnectionType: + connection.system_type # Makes sure all ConnectionTypes have been added to system_type mapping + + def test_connection_type_system_type_invalid(self): + with pytest.raises(NotImplementedError): + ConnectionType("nonmapped_type").system_type def test_system_key(self, db, connection_config, system): assert connection_config.system_key == connection_config.name From 72cd9b3fb1b5886767458b060bddcb8e33e0960e Mon Sep 17 00:00:00 2001 From: Catherine Smith Date: Wed, 15 Jan 2025 20:59:08 +0100 Subject: [PATCH 16/50] LA-174: Adds max rows limit config to privacy req csv download (#5671) --- CHANGELOG.md | 1 + .../privacy-requests/privacy-requests.slice.ts | 6 +++++- .../v1/endpoints/privacy_request_endpoints.py | 16 ++++++++++++++++ src/fides/config/admin_ui_settings.py | 4 ++++ tests/fixtures/application_fixtures.py | 8 ++++++++ .../endpoints/test_privacy_request_endpoints.py | 13 +++++++++++++ 6 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e76b4dd0b8..7543740523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Removed primary key requirements for BigQuery and Postgres erasures [#5591](https://github.com/ethyca/fides/pull/5591) - Updated `DBCache` model so setting cache value always updates the updated_at field [#5669](https://github.com/ethyca/fides/pull/5669) - Changed sizes of buttons in table headers [#5654](https://github.com/ethyca/fides/pull/5654) +- Adds new config for max number of rows in DSR download through Admin-UI [#5671](https://github.com/ethyca/fides/pull/5671) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts index b69cf28f20..aa5c1f4fe3 100644 --- a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts +++ b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts @@ -110,8 +110,12 @@ export const requestCSVDownload = async ({ }, }, ) - .then((response) => { + .then(async (response) => { if (!response.ok) { + if (response.status === 400) { + const errorData = await response.json(); + throw new Error(errorData.detail || "Bad request error"); + } throw new Error("Got a bad response from the server"); } return response.blob(); diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index bfe42a1455..0a39866924 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -672,6 +672,21 @@ def attach_resume_instructions(privacy_request: PrivacyRequest) -> None: ) +def _validate_result_size(query: Query) -> None: + """ + Validates the result size is less than maximum allowed by settings. + Raises an HTTPException if result size is greater than maximum. + Result size is determined by running an up-front "count" query. + """ + row_count = query.count() + max_rows = CONFIG.admin_ui.max_privacy_request_download_rows + if row_count > max_rows: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"Requested privacy request report would contain {row_count} rows. A maximum of {max_rows} rows is permitted. Please narrow your date range and try again.", + ) + + def _shared_privacy_request_search( *, db: Session, @@ -740,6 +755,7 @@ def _shared_privacy_request_search( query = _sort_privacy_request_queryset(query, sort_field, sort_direction) if download_csv: + _validate_result_size(query) # Returning here if download_csv param was specified logger.info("Downloading privacy requests as csv") return privacy_request_csv_download(db, query) diff --git a/src/fides/config/admin_ui_settings.py b/src/fides/config/admin_ui_settings.py index 67cacba456..cd50a01675 100644 --- a/src/fides/config/admin_ui_settings.py +++ b/src/fides/config/admin_ui_settings.py @@ -17,4 +17,8 @@ class AdminUISettings(FidesSettings): url: SerializeAsAny[Optional[AnyHttpUrlStringRemovesSlash]] = Field( default=None, description="The base URL for the Admin UI." ) + max_privacy_request_download_rows: int = Field( + default=100000, + description="The maximum number of rows permitted to be returned in a privacy request report download", + ) model_config = SettingsConfigDict(env_prefix="FIDES__ADMIN_UI__") diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index d030919aed..19075f09de 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -3498,6 +3498,14 @@ def allow_custom_privacy_request_fields_in_request_execution_disabled(): ) +@pytest.fixture(scope="function") +def set_max_privacy_request_download_rows(): + original_value = CONFIG.admin_ui.max_privacy_request_download_rows + CONFIG.admin_ui.max_privacy_request_download_rows = 1 + yield + CONFIG.admin_ui.max_privacy_request_download_rows = original_value + + @pytest.fixture(scope="function") def subject_request_download_ui_enabled(): original_value = CONFIG.security.subject_request_download_ui_enabled diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index c4f386d7b0..2cc83707b3 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -1951,6 +1951,19 @@ def test_get_privacy_requests_csv_format( privacy_request.delete(db) + def test_get_privacy_requests_csv_format_max_rows_limit( + self, + db, + generate_auth_header, + api_client, + url, + privacy_requests, + set_max_privacy_request_download_rows, + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get(url + f"?download_csv=True", headers=auth_header) + assert 400 == response.status_code + def test_get_requires_input_privacy_request_resume_info( self, db, privacy_request, generate_auth_header, api_client, url ): From b3cb8d0e73e96afb7232da94c48602cccfd937e6 Mon Sep 17 00:00:00 2001 From: Andres Torres Date: Wed, 15 Jan 2025 14:51:49 -0600 Subject: [PATCH 17/50] Tests quick fix: tests/ops/models/test_connectionconfig.py (#5672) --- tests/ops/models/test_connectionconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ops/models/test_connectionconfig.py b/tests/ops/models/test_connectionconfig.py index b24a43dbf3..347abf3836 100644 --- a/tests/ops/models/test_connectionconfig.py +++ b/tests/ops/models/test_connectionconfig.py @@ -146,7 +146,7 @@ def test_connection_type_human_readable(self): connection.human_readable # Makes sure all ConnectionTypes have been added to human_readable mapping def test_connection_type_human_readable_invalid(self): - with pytest.raises(NotImplementedError): + with pytest.raises(ValueError): ConnectionType("nonmapped_type").human_readable def test_connection_type_system_type(self): @@ -154,7 +154,7 @@ def test_connection_type_system_type(self): connection.system_type # Makes sure all ConnectionTypes have been added to system_type mapping def test_connection_type_system_type_invalid(self): - with pytest.raises(NotImplementedError): + with pytest.raises(ValueError): ConnectionType("nonmapped_type").system_type def test_system_key(self, db, connection_config, system): From c6fa9d35947c0ac319d3a25a1d3a35b505bbe304 Mon Sep 17 00:00:00 2001 From: Lucano Vera Date: Wed, 15 Jan 2025 18:06:52 -0300 Subject: [PATCH 18/50] Add missing install python 3.10 (#5674) --- .github/workflows/backend_checks.yml | 18 ++++++++++++++++++ .github/workflows/cypress_e2e.yml | 6 ++++-- .github/workflows/publish_docker.yaml | 8 +++++++- .github/workflows/publish_docs.yaml | 9 ++++++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index 065a8ef920..ffffc56c0d 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -284,6 +284,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" + - name: Install Nox run: pip install nox>=2022 @@ -340,6 +346,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" + - name: Install Nox run: pip install nox>=2022 @@ -426,6 +438,12 @@ jobs: - name: Load image run: docker load --input /tmp/python-${{ matrix.python_version }}.tar + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" + - name: Install Nox run: pip install nox>=2022 diff --git a/.github/workflows/cypress_e2e.yml b/.github/workflows/cypress_e2e.yml index 7da0f97836..9ff206184c 100644 --- a/.github/workflows/cypress_e2e.yml +++ b/.github/workflows/cypress_e2e.yml @@ -15,6 +15,7 @@ env: # Docker auth with read-only permissions. DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_RO_TOKEN: ${{ secrets.DOCKER_RO_TOKEN }} + DEFAULT_PYTHON_VERSION: "3.10.13" jobs: Cypress-E2E: @@ -26,10 +27,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Setup Python 3.10 + - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" - name: Install Nox run: pip install nox>=2022 diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index 943f0514cc..6b86264994 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -11,6 +11,7 @@ env: # Docker auth with read-write (publish) permissions. Set as env in workflow root as auth is required in multiple jobs. DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + DEFAULT_PYTHON_VERSION: "3.10.13" jobs: ParseTags: @@ -71,6 +72,12 @@ jobs: with: fetch-depth: 0 # This is required to properly tag images + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" + - name: Login to DockerHub uses: docker/login-action@v3 with: @@ -104,7 +111,6 @@ jobs: if: needs.ParseTags.outputs.alpha_tag == 'true' || needs.ParseTags.outputs.beta_tag == 'true' run: nox -s "push(${{ matrix.application }},prerelease)" -- git_tag - NotifyRedeploy: runs-on: ubuntu-latest needs: Push diff --git a/.github/workflows/publish_docs.yaml b/.github/workflows/publish_docs.yaml index 1e46500458..7a096b7f06 100644 --- a/.github/workflows/publish_docs.yaml +++ b/.github/workflows/publish_docs.yaml @@ -10,6 +10,7 @@ on: env: TAG: ${{ github.event.release.tag_name }} PROD_PUBLISH: true + DEFAULT_PYTHON_VERSION: "3.10.13" jobs: publish_docs: @@ -19,9 +20,11 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - name: Set Up Python + uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + cache: "pip" - name: Echo the Release Tag run: echo ${{ env.TAG }} @@ -31,7 +34,7 @@ jobs: - name: Install Docs Requirements run: pip install -r docs/fides/requirements.txt - + - name: Install fides run: pip install -e ./[all] From 71aa1a2d79b965db9ae892c730d28cdf5f43201e Mon Sep 17 00:00:00 2001 From: Lucano Vera Date: Wed, 15 Jan 2025 18:42:25 -0300 Subject: [PATCH 19/50] Fix order of steps of CI backend checks (#5675) --- .github/workflows/backend_checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index ffffc56c0d..61e279f246 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -438,6 +438,9 @@ jobs: - name: Load image run: docker load --input /tmp/python-${{ matrix.python_version }}.tar + - name: Checkout + uses: actions/checkout@v4 + - name: Set Up Python uses: actions/setup-python@v5 with: @@ -447,9 +450,6 @@ jobs: - name: Install Nox run: pip install nox>=2022 - - name: Checkout - uses: actions/checkout@v4 - - name: Get Vault Token uses: hashicorp/vault-action@v2.5.0 with: From 46cb2242ac1a84718871ff4898a6fca8a214ab4e Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Thu, 16 Jan 2025 08:01:34 -0500 Subject: [PATCH 20/50] bump `external-datastores` job timeout from 20 to 30 min (#5677) --- .github/workflows/backend_checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index 61e279f246..5d3705ae3a 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -330,7 +330,7 @@ jobs: matrix: python_version: ["3.9.18", "3.10.13"] runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 # In PRs run with the "unsafe" label, or run on a "push" event to main if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' steps: From b7fe7393559720a768be74f23ab3f6a4e59ada1e Mon Sep 17 00:00:00 2001 From: Roger Plotz <115798183+Roger-Ethyca@users.noreply.github.com> Date: Thu, 16 Jan 2025 09:10:21 -0500 Subject: [PATCH 21/50] Update changelog release 2.53.0 (#5678) --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7543740523..d5379f3972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,12 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - https://github.com/ethyca/fides/labels/high-risk: to indicate that a change is a "high-risk" change that could potentially lead to unanticipated regressions or degradations - https://github.com/ethyca/fides/labels/db-migration: to indicate that a given change includes a DB migration -## [Unreleased](https://github.com/ethyca/fides/compare/2.52.0...main) +## [Unreleased](https://github.com/ethyca/fides/compare/2.53.0...main) + + + + +## [2.53.0](https://github.com/ethyca/fides/compare/2.52.0...2.53.0) ### Added - Added Action Center MVP behind new feature flag [#5622](https://github.com/ethyca/fides/pull/5622) From 7bb06934fc62d6f5d3af2b4d1e06cd98d0a10a55 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Thu, 16 Jan 2025 11:31:11 -0700 Subject: [PATCH 22/50] Convert `rem` to base font size (#5673) --- CHANGELOG.md | 1 + clients/fides-js/src/components/fides.css | 26 +++++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5379f3972..df848dd294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Updated `DBCache` model so setting cache value always updates the updated_at field [#5669](https://github.com/ethyca/fides/pull/5669) - Changed sizes of buttons in table headers [#5654](https://github.com/ethyca/fides/pull/5654) - Adds new config for max number of rows in DSR download through Admin-UI [#5671](https://github.com/ethyca/fides/pull/5671) +- Added CSS variable to FidesJS: `--fides-base-font-size: 16px` for better consistency. Overriding this variable with "1rem" will mimic legacy behavior. [#5673](https://github.com/ethyca/fides/pull/5673) https://github.com/ethyca/fides/labels/high-risk ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/fides-js/src/components/fides.css b/clients/fides-js/src/components/fides.css index 3eb261516b..79e1d866df 100644 --- a/clients/fides-js/src/components/fides.css +++ b/clients/fides-js/src/components/fides.css @@ -56,11 +56,11 @@ /* Everything else */ --fides-overlay-font-family: Inter, sans-serif; - --8px: 0.5rem; - --12px: 0.75rem; - --14px: 0.875rem; - --15px: 0.9375rem; - --16px: 1rem; + --fides-base-font-size: 16px; + --8px: calc(var(--fides-base-font-size) * 0.5); + --12px: calc(var(--fides-base-font-size) * 0.75); + --14px: calc(var(--fides-base-font-size) * 0.875); + --16px: calc(var(--fides-base-font-size) * 1); --fides-overlay-font-size-body-xs: var(--8px); --fides-overlay-font-size-body-small: var(--12px); --fides-overlay-font-size-body: var(--14px); @@ -115,9 +115,7 @@ div#fides-overlay-wrapper * { font-family: var(--fides-overlay-font-family); font-size: var(--fides-overlay-font-size-body); white-space: pre-line; - - /* CSS reset values, adapted from https://www.joshwcomeau.com/css/custom-css-reset/ */ - line-height: calc(1em + 0.4rem); + line-height: 1.4em; -webkit-font-smoothing: antialiased; } @@ -737,7 +735,7 @@ div#fides-overlay-wrapper .fides-toggle .fides-toggle-display { display: inline-block; min-width: 8px; height: 8px; - margin-right: calc(0.5rem + 2px); + margin-right: calc(var(--8px) + 2px); /* half font size + border width */ content: ""; transition: transform 0.12s ease-in-out; transform: translateY(-2px) rotate(135deg); @@ -1065,7 +1063,7 @@ div#fides-overlay-wrapper .fides-i18n-pseudo-button { display: flex; flex-direction: column; gap: 1px; - max-height: 7rem; + max-height: calc(var(--fides-base-font-size) * 7); transition: height 0.5s; } @@ -1079,7 +1077,7 @@ div#fides-overlay-wrapper .fides-i18n-pseudo-button { .fides-i18n-menu:hover .fides-i18n-popover, .fides-i18n-menu:focus-within .fides-i18n-popover { - min-width: 9rem; + min-width: calc(var(--fides-base-font-size) * 9); height: auto; border: 1px solid var(--fides-overlay-primary-color); background-color: var(--fides-overlay-background-dark-color); @@ -1095,7 +1093,7 @@ button.fides-banner-button.fides-menu-item { width: 100%; text-align: left; margin: 0; - padding-left: 1.5rem; + padding-left: calc(var(--fides-base-font-size) * 1.5); } button.fides-banner-button.fides-menu-item[aria-pressed="true"] { @@ -1106,8 +1104,8 @@ button.fides-banner-button.fides-menu-item[aria-pressed="true"] { button.fides-banner-button.fides-menu-item[aria-pressed="true"]::before { content: "\2713"; display: inline-block; - margin-right: 0.25rem; - margin-left: -1rem; + margin-right: calc(var(--fides-base-font-size) * 0.25); + margin-left: calc(var(--fides-base-font-size) * -1); } button.fides-banner-button.fides-menu-item:not([aria-pressed="true"]):hover { From af64ee95ed66395289b7b4c1c9b79113d7e8a181 Mon Sep 17 00:00:00 2001 From: Jason Gill Date: Thu, 16 Jan 2025 15:11:40 -0700 Subject: [PATCH 23/50] Action Center: Webmonitor Assets View (#5676) --- .../admin-ui/cypress/e2e/action-center.cy.ts | 153 ++++++- .../system-aggregate-results.json | 4 +- .../activity-center/system-asset-results.json | 400 ++++++++++++++++++ clients/admin-ui/cypress/support/stubs.ts | 14 +- .../features/common/dropdown/SystemSelect.tsx | 57 +++ .../src/features/common/hooks/useAlert.tsx | 9 +- .../src/features/common/nav/v2/routes.ts | 2 + .../features/common/table/v2/FidesCell.tsx | 4 +- .../features/common/table/v2/FidesTable.tsx | 7 +- .../src/features/common/table/v2/util.ts | 4 +- .../ClassificationCategoryBadge.tsx | 2 +- .../action-center/action-center.slice.ts | 48 +++ .../hooks/useDiscoveredAssetsColumns.tsx | 113 +++++ .../useDiscoveredSystemAggregateColumns.tsx | 14 +- .../tables/DiscoveredAssetsTable.tsx | 238 +++++++++++ .../tables/DiscoveredSystemAggregateTable.tsx | 26 +- .../cells/DiscoveredAssetActionsCell.tsx | 76 ++++ .../tables/cells/DiscoveryStatusBadgeCell.tsx | 42 ++ .../action-center/tables/cells/SystemCell.tsx | 83 ++++ .../action-center/{types.ts => types.d.ts} | 1 + .../discovery-detection.slice.ts | 6 +- .../[monitorId]/[systemId]/index.tsx | 35 ++ .../action-center/[monitorId]/index.tsx | 2 +- .../api/models/StagedResourceAPIResponse.ts | 4 + clients/fidesui/src/hoc/CustomSelect.tsx | 4 +- 25 files changed, 1307 insertions(+), 41 deletions(-) create mode 100644 clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-results.json create mode 100644 clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/hooks/useDiscoveredAssetsColumns.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/DiscoveredAssetsTable.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveredAssetActionsCell.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/DiscoveryStatusBadgeCell.tsx create mode 100644 clients/admin-ui/src/features/data-discovery-and-detection/action-center/tables/cells/SystemCell.tsx rename clients/admin-ui/src/features/data-discovery-and-detection/action-center/{types.ts => types.d.ts} (98%) create mode 100644 clients/admin-ui/src/pages/data-discovery/action-center/[monitorId]/[systemId]/index.tsx diff --git a/clients/admin-ui/cypress/e2e/action-center.cy.ts b/clients/admin-ui/cypress/e2e/action-center.cy.ts index 1f816ae3b3..3cc201c343 100644 --- a/clients/admin-ui/cypress/e2e/action-center.cy.ts +++ b/clients/admin-ui/cypress/e2e/action-center.cy.ts @@ -51,10 +51,10 @@ describe("Action center", () => { const integrationMonitorKey = "My_New_BQ_Monitor"; beforeEach(() => { cy.visit(ACTION_CENTER_ROUTE); + cy.wait("@getMonitorResults"); }); it("should render the current monitor results", () => { cy.get("[data-testid='Action center']").should("exist"); - cy.wait("@getMonitorResults"); cy.get("[data-testid*='monitor-result-']").should("have.length", 3); cy.get("[data-testid^='monitor-result-']").each((result) => { const monitorKey = result @@ -83,7 +83,6 @@ describe("Action center", () => { ); }); it("should have appropriate actions for web monitors", () => { - cy.wait("@getMonitorResults"); // Add button // TODO: [HJ-337] uncomment when Add button is implemented // cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); @@ -95,7 +94,6 @@ describe("Action center", () => { ); }); it.skip("Should have appropriate actions for Integrations monitors", () => { - cy.wait("@getMonitorResults"); // Classify button cy.getByTestId(`review-button-${integrationMonitorKey}`).should( "have.attr", @@ -106,7 +104,6 @@ describe("Action center", () => { cy.getByTestId(`ignore-button-${integrationMonitorKey}`).should("exist"); }); it.skip("Should have appropriate actions for SSO monitors", () => { - cy.wait("@getMonitorResults"); // Add button cy.getByTestId(`add-button-${webMonitorKey}`).should("exist"); // Ignore button @@ -121,6 +118,7 @@ describe("Action center", () => { const webMonitorKey = "my_web_monitor_1"; beforeEach(() => { cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}`); + cy.wait("@getSystemAggregateResults"); }); it("should display a breadcrumb", () => { cy.getByTestId("page-breadcrumb").within(() => { @@ -131,26 +129,28 @@ describe("Action center", () => { }); }); it("should render the aggregated system results in a table", () => { - cy.wait("@getSystemAggregateResults"); + cy.getByTestId("search-bar").should("exist"); + cy.getByTestId("pagination-btn").should("exist"); cy.getByTestId("column-system_name").should("exist"); cy.getByTestId("column-total_updates").should("exist"); - cy.getByTestId("column-data_use").should("exist"); + // TODO: [HJ-356] uncomment when data use column is implemented + // cy.getByTestId("column-data_use").should("exist"); cy.getByTestId("column-locations").should("exist"); cy.getByTestId("column-domains").should("exist"); - cy.getByTestId("column-actions").should("exist"); - cy.getByTestId("search-bar").should("exist"); - cy.getByTestId("pagination-btn").should("exist"); + // TODO: [HJ-343] uncomment when actions column is implemented + // cy.getByTestId("column-actions").should("exist"); cy.getByTestId("row-0-col-system_name").within(() => { cy.getByTestId("change-icon").should("exist"); // new result cy.contains("Uncategorized assets").should("exist"); }); - // data use column should be empty for uncategorized assets + // TODO: [HJ-356] uncomment when data use column is implemented + /* // data use column should be empty for uncategorized assets cy.getByTestId("row-0-col-data_use").children().should("have.length", 0); cy.getByTestId("row-1-col-system_name").within(() => { cy.getByTestId("change-icon").should("not.exist"); // existing result cy.contains("Google Tag Manager").should("exist"); - }); - // TODO: data use column should not be empty for other assets + }); */ + // TODO: [HJ-356] data use column should not be empty for other assets // cy.getByTestId("row-1-col-data_use").children().should("not.have.length", 0); // multiple locations @@ -166,10 +166,129 @@ describe("Action center", () => { "analytics.google.com", ); }); - // it("should navigate to table view on row click", () => { - // cy.getByTestId("row-1").click(); - // cy.url().should("contain", "fds.1046"); - // cy.getByTestId("page-breadcrumb").should("contain", "fds.1046"); - // }); + it("should navigate to table view on row click", () => { + cy.getByTestId("row-1").click(); + cy.url().should( + "contain", + "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + ); + cy.getByTestId("page-breadcrumb").should( + "contain", + "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + ); + }); + }); + + describe("Action center system assets results", () => { + const webMonitorKey = "my_web_monitor_1"; + const systemId = "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88"; + beforeEach(() => { + cy.visit(`${ACTION_CENTER_ROUTE}/${webMonitorKey}/${systemId}`); + cy.wait("@getSystemAssetResults"); + }); + it("should render asset results view", () => { + cy.getByTestId("page-breadcrumb").should("contain", systemId); + cy.getByTestId("search-bar").should("exist"); + cy.getByTestId("pagination-btn").should("exist"); + cy.getByTestId("bulk-actions-menu").should("be.disabled"); + + // table columns + cy.getByTestId("column-select").should("exist"); + cy.getByTestId("column-name").should("exist"); + cy.getByTestId("column-resource_type").should("exist"); + cy.getByTestId("column-system").should("exist"); + // TODO: [HJ-369] uncomment when data use column is implemented + // cy.getByTestId("column-data_use").should("exist"); + cy.getByTestId("column-locations").should("exist"); + cy.getByTestId("column-domain").should("exist"); + // TODO: [HJ-344] uncomment when Discovery column is implemented + /* cy.getByTestId("column-with_consent").should("exist"); + cy.getByTestId("row-4-col-with_consent") + .contains("Without consent") + .realHover(); + cy.get(".ant-tooltip-inner").should("contain", "January"); */ + cy.getByTestId("column-actions").should("exist"); + cy.getByTestId("row-0-col-actions").within(() => { + cy.getByTestId("add-btn").should("exist"); + cy.getByTestId("ignore-btn").should("exist"); + }); + }); + it.skip("should allow adding a system on uncategorized assets", () => { + // TODO: uncategorized assets are not yet available for testing + }); + it("should allow editing a system on categorized assets", () => { + cy.getByTestId("page-breadcrumb").should("contain", systemId); // little hack to make sure the systemId is available before proceeding + cy.getByTestId("row-3-col-system").within(() => { + cy.getByTestId("system-badge").click(); + }); + cy.wait("@getSystemsPaginated"); + cy.getByTestId("system-select").antSelect("Fidesctl System"); + cy.wait("@setAssetSystem"); + cy.getByTestId("system-select").should("not.exist"); + + // Wait for previous UI animations to reset or Cypress chokes on the next part + // eslint-disable-next-line cypress/no-unnecessary-waiting + cy.wait(100); + + // Now test with search + cy.getByTestId("row-2-col-system").within(() => { + cy.getByTestId("system-badge").click(); + cy.getByTestId("system-select").find("input").type("demo m"); + cy.wait("@getSystemsWithSearch").then((interception) => { + expect(interception.request.query.search).to.eq("demo m"); + }); + }); + cy.wait("@getSystemsPaginated"); + cy.getByTestId("system-select").antSelect("Demo Marketing System"); + cy.wait("@setAssetSystem"); + cy.getByTestId("success-alert").should("exist"); + cy.getByTestId("system-select").should("not.exist"); + }); + it("should add individual assets", () => { + cy.getByTestId("row-0-col-actions").within(() => { + cy.getByTestId("add-btn").click({ force: true }); + }); + cy.wait("@addAssets"); + cy.getByTestId("success-alert").should("exist"); + }); + it("should ignore individual assets", () => { + cy.getByTestId("row-0-col-actions").within(() => { + cy.getByTestId("ignore-btn").click({ force: true }); + }); + cy.wait("@ignoreAssets"); + cy.getByTestId("success-alert").should("exist"); + }); + it("should bulk add assets", () => { + cy.getByTestId("bulk-actions-menu").should("be.disabled"); + cy.getByTestId("row-0-col-select").find("label").click(); + cy.getByTestId("row-2-col-select").find("label").click(); + cy.getByTestId("row-3-col-select").find("label").click(); + cy.getByTestId("selected-count").should("contain", "3 selected"); + cy.getByTestId("bulk-actions-menu").should("not.be.disabled"); + cy.getByTestId("bulk-actions-menu").click(); + cy.getByTestId("bulk-add").click(); + cy.wait("@addAssets"); + cy.getByTestId("success-alert").should("exist"); + }); + it("should bulk ignore assets", () => { + cy.getByTestId("bulk-actions-menu").should("be.disabled"); + cy.getByTestId("row-0-col-select").find("label").click(); + cy.getByTestId("row-2-col-select").find("label").click(); + cy.getByTestId("row-3-col-select").find("label").click(); + cy.getByTestId("selected-count").should("contain", "3 selected"); + cy.getByTestId("bulk-actions-menu").should("not.be.disabled"); + cy.getByTestId("bulk-actions-menu").click(); + cy.getByTestId("bulk-ignore").click(); + cy.wait("@ignoreAssets"); + cy.getByTestId("success-alert").should("exist"); + }); + it.skip("should add all assets", () => { + // TODO: [HJ-343] unskip when add all is implemented + cy.getByTestId("add-all").click(); + cy.getByTestId("add-all").should("have.class", "ant-btn-loading"); + cy.wait("@addAssets"); + cy.url().should("not.contain", systemId); + cy.getByTestId("success-alert").should("exist"); + }); }); }); diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json index 3bcc4b2328..566880f701 100644 --- a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-aggregate-results.json @@ -40,9 +40,9 @@ ] }, { - "id": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "id": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", "name": "Google Tag Manager", - "system_key": "system_key-72649f03-7a30-4758-9772-e74fca3b6788", + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", "vendor_id": "fds.1046", "total_updates": 10, "locations": ["USA"], diff --git a/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-results.json b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-results.json new file mode 100644 index 0000000000..e0bbb4d241 --- /dev/null +++ b/clients/admin-ui/cypress/fixtures/detection-discovery/activity-center/system-asset-results.json @@ -0,0 +1,400 @@ +{ + "items": [ + { + "urn": "my_web_monitor_1.GET.td.doubleclick.net.https://td.doubleclick.net/td/rul/11020051272", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "11020051272", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.504650Z", + "diff_status": "addition", + "domain": "td.doubleclick.net", + "parent": "https://www.googletagmanager.com/gtag/js?id=AW-11020051272&l=dataLayer&cx=c>m=45je51d0v9134906606za200", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://td.doubleclick.net/td/rul/11020051272", + "mime_type": "text/html", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.GET.td.doubleclick.net.https://td.doubleclick.net/td/rul/697301175", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "697301175", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.499796Z", + "diff_status": "addition", + "domain": "td.doubleclick.net", + "parent": "https://www.googletagmanager.com/gtag/js?id=AW-697301175&l=dataLayer&cx=c>m=45je51d0v9134906606za200", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://td.doubleclick.net/td/rul/697301175", + "mime_type": "text/html", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.POST.www.google.com.https://www.google.com/ccm/collect", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "collect", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.423885Z", + "diff_status": "addition", + "domain": "www.google.com", + "parent": "https://www.googletagmanager.com/gtm.js?id=GTM-PZHT655", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.google.com/ccm/collect", + "size": 0, + "method": "POST" + }, + { + "urn": "my_web_monitor_1.GET.www.googletagmanager.com.https://www.googletagmanager.com/gtag/destination", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "destination", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.427163Z", + "diff_status": "addition", + "domain": "www.googletagmanager.com", + "parent": "https://www.googletagmanager.com/gtm.js?id=GTM-PZHT655", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.googletagmanager.com/gtag/destination", + "mime_type": "application/javascript", + "size": 103688, + "method": "GET" + }, + { + "urn": "my_web_monitor_1.GET.www.googletagmanager.com.https://www.googletagmanager.com/gtm.js", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "gtm.js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.403801Z", + "diff_status": "addition", + "domain": "www.googletagmanager.com", + "parent": null, + "parent_domain": null, + "locations": ["USA"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.googletagmanager.com/gtm.js", + "mime_type": "application/javascript", + "size": 110055, + "method": "GET" + }, + { + "urn": "my_web_monitor_1.GET.www.google.com.https://www.google.com/pagead/1p-user-list/11020051272/", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "https://www.google.com/pagead/1p-user-list/11020051272/", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.513011Z", + "diff_status": "addition", + "domain": "www.google.com", + "parent": "https://googleads.g.doubleclick.net/pagead/viewthroughconversion/11020051272/?random=1737048446598&cv=11&fst=1737048446598&bg=ffffff&guid=ON&async=1>m=45be51d0v897313980za200zb9134906606&gcd=13l3l3l3l1l1&dma=0&tag_exp=101925629~102067555~102067808~102081485~102123608~102198178&u_w=1536&u_h=864&url=https%3A%2F%2Fethyca.com%2F&hn=www.googleadservices.com&frm=0&tiba=Ethyca%20%7C%20Home&did=dZTQ1Zm&gdid=dZTQ1Zm&npa=0&pscdl=noapi&auid=1657606680.1737048443&uaa=x86&uab=64&uafvl=Google%2520Chrome%3B131.0.6778.109%7CChromium%3B131.0.6778.109%7CNot_A%2520Brand%3B24.0.0.0&uamb=0&uam=&uap=Windows&uapv=10.0.0&uaw=0&fledge=1&data=event%3Dgtag.config&rfmt=3&fmt=4", + "parent_domain": "googleads.g.doubleclick.net", + "locations": ["USA", "Canada"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.google.com/pagead/1p-user-list/11020051272/", + "mime_type": "image/gif", + "size": 108, + "method": "GET" + }, + { + "urn": "my_web_monitor_1.GET.www.google.com.https://www.google.com/pagead/1p-user-list/697301175/", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "https://www.google.com/pagead/1p-user-list/697301175/", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.509468Z", + "diff_status": "addition", + "domain": "www.google.com", + "parent": "https://googleads.g.doubleclick.net/pagead/viewthroughconversion/697301175/?random=1737048446477&cv=11&fst=1737048446477&bg=ffffff&guid=ON&async=1>m=45be51d0v895464401za200zb9134906606&gcd=13l3l3l3l1l1&dma=0&tag_exp=101925629~102067555~102067808~102081485~102123607~102198178&u_w=1536&u_h=864&url=https%3A%2F%2Fethyca.com%2F&hn=www.googleadservices.com&frm=0&tiba=Ethyca%20%7C%20Home&did=dZTQ1Zm&gdid=dZTQ1Zm&npa=0&pscdl=noapi&auid=1657606680.1737048443&uaa=x86&uab=64&uafvl=Google%2520Chrome%3B131.0.6778.109%7CChromium%3B131.0.6778.109%7CNot_A%2520Brand%3B24.0.0.0&uamb=0&uam=&uap=Windows&uapv=10.0.0&uaw=0&fledge=1&data=event%3Dgtag.config&rfmt=3&fmt=4", + "parent_domain": "googleads.g.doubleclick.net", + "locations": ["USA"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.google.com/pagead/1p-user-list/697301175/", + "mime_type": "image/gif", + "size": 455, + "method": "GET" + }, + { + "urn": "my_web_monitor_1..doubleclick.net.IDE", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "IDE", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.256532Z", + "diff_status": "addition", + "domain": ".doubleclick.net", + "parent": "https://www.googletagmanager.com/gtag/destination?id=AW-11020051272&l=dataLayer&cx=c>m=45He51d0v810253976za200", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Cookie", + "system": "Google Tag Manager", + "value": "AHWqTUkbgHZ9_6gV1ZYVnfFx8h3-wbUSqy-qmOM6ay-n72yyvwky-VDu7PeobUgP", + "path": "/" + }, + { + "urn": "my_web_monitor_1.GET.www.googletagmanager.com.https://www.googletagmanager.com/gtag/js", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "js", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.493601Z", + "diff_status": "addition", + "domain": "www.googletagmanager.com", + "parent": "https://www.googletagmanager.com/gtag/js?id=G-B356CF15GS", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.googletagmanager.com/gtag/js", + "mime_type": "application/javascript", + "size": 103387, + "method": "GET" + }, + { + "urn": "my_web_monitor_1.GET.td.doubleclick.net.https://td.doubleclick.net/td/ga/rul", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "rul", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.420514Z", + "diff_status": "addition", + "domain": "td.doubleclick.net", + "parent": "https://www.googletagmanager.com/gtag/js?id=G-B356CF15GS", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://td.doubleclick.net/td/ga/rul", + "cookies": [ + "test_cookie=CheckForPermission; expires=Thu, 16-Jan-2025 17:42:23 GMT; path=/; domain=.doubleclick.net; Secure; SameSite=none" + ], + "mime_type": "text/html", + "method": "GET" + }, + { + "urn": "my_web_monitor_1.GET.www.googletagmanager.com.https://www.googletagmanager.com/static/service_worker/51g0/sw_iframe.html", + "user_assigned_data_categories": [], + "system_key": "system_key-8fe42cdb-af2e-4b9e-9b38-f75673180b88", + "name": "sw_iframe.html", + "description": null, + "monitor_config_id": "my_web_monitor_1", + "updated_at": "2025-01-16T17:27:31.437802Z", + "diff_status": "addition", + "domain": "www.googletagmanager.com", + "parent": "https://www.googletagmanager.com/gtm.js?id=GTM-PZHT655", + "parent_domain": "www.googletagmanager.com", + "locations": ["USA", "Canada"], + "with_consent": false, + "data_uses": [], + "vendor_id": "fds.1046", + "system_id": "ctl_e0ac4744-0f2e-4feb-b281-5877f9e464d1", + "source_modified": null, + "classifications": [], + "child_diff_statuses": {}, + "database_name": null, + "schema_name": null, + "parent_table_urn": null, + "table_name": null, + "data_type": null, + "fields": [], + "num_rows": null, + "tables": [], + "schemas": [], + "resource_type": "Browser Request", + "system": "Google Tag Manager", + "base_url": "https://www.googletagmanager.com/static/service_worker/51g0/sw_iframe.html", + "mime_type": "text/html", + "method": "GET" + } + ], + "total": 11, + "page": 1, + "size": 25, + "pages": 1 +} diff --git a/clients/admin-ui/cypress/support/stubs.ts b/clients/admin-ui/cypress/support/stubs.ts index 6c3109f1e9..df518408db 100644 --- a/clients/admin-ui/cypress/support/stubs.ts +++ b/clients/admin-ui/cypress/support/stubs.ts @@ -516,9 +516,21 @@ export const stubActionCenter = () => { }).as("getMonitorResults"); cy.intercept( "GET", - "/api/v1//plus/discovery-monitor/system-aggregate-results*", + "/api/v1/plus/discovery-monitor/system-aggregate-results*", { fixture: "detection-discovery/activity-center/system-aggregate-results", }, ).as("getSystemAggregateResults"); + cy.intercept("GET", "/api/v1/plus/discovery-monitor/*/results*", { + fixture: "detection-discovery/activity-center/system-asset-results", + }).as("getSystemAssetResults"); + cy.intercept("POST", "/api/v1/plus/discovery-monitor/mute*", { + response: 200, + }).as("ignoreAssets"); + cy.intercept("POST", "/api/v1/plus/discovery-monitor/promote*", { + response: 200, + }).as("addAssets"); + cy.intercept("PATCH", "/api/v1/plus/discovery-monitor/*/results", { + response: 200, + }).as("setAssetSystem"); }; diff --git a/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx b/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx new file mode 100644 index 0000000000..34501790e3 --- /dev/null +++ b/clients/admin-ui/src/features/common/dropdown/SystemSelect.tsx @@ -0,0 +1,57 @@ +import { AntSelect as Select, AntSelectProps as SelectProps } from "fidesui"; +import { useCallback, useMemo, useState } from "react"; + +import { useGetSystemsQuery } from "~/features/system/system.slice"; + +import { debounce } from "../utils"; + +const OPTIONS_LIMIT = 25; + +interface SystemSelectProps + extends Omit< + SelectProps, + "options" | "showSearch" | "filterOption" | "onSearch" + > {} + +export const SystemSelect = ({ ...props }: SystemSelectProps) => { + const [searchValue, setSearchValue] = useState(); + const { data, isFetching } = useGetSystemsQuery({ + page: 1, + size: OPTIONS_LIMIT, + search: searchValue || undefined, + }); + + const options = data?.items?.map((system) => ({ + value: system.fides_key, + label: system.name, + })); + + const handleSearch = useCallback( + (search: string) => { + if (search?.length > 1) { + setSearchValue(search); + } + if (search?.length === 0) { + setSearchValue(undefined); + } + }, + [setSearchValue], + ); + + const onSearch = useMemo(() => debounce(handleSearch, 300), [handleSearch]); + + return ( + +