From 72cd9b3fb1b5886767458b060bddcb8e33e0960e Mon Sep 17 00:00:00 2001 From: Catherine Smith Date: Wed, 15 Jan 2025 20:59:08 +0100 Subject: [PATCH] LA-174: Adds max rows limit config to privacy req csv download (#5671) --- CHANGELOG.md | 1 + .../privacy-requests/privacy-requests.slice.ts | 6 +++++- .../v1/endpoints/privacy_request_endpoints.py | 16 ++++++++++++++++ src/fides/config/admin_ui_settings.py | 4 ++++ tests/fixtures/application_fixtures.py | 8 ++++++++ .../endpoints/test_privacy_request_endpoints.py | 13 +++++++++++++ 6 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e76b4dd0b8..7543740523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o - Removed primary key requirements for BigQuery and Postgres erasures [#5591](https://github.com/ethyca/fides/pull/5591) - Updated `DBCache` model so setting cache value always updates the updated_at field [#5669](https://github.com/ethyca/fides/pull/5669) - Changed sizes of buttons in table headers [#5654](https://github.com/ethyca/fides/pull/5654) +- Adds new config for max number of rows in DSR download through Admin-UI [#5671](https://github.com/ethyca/fides/pull/5671) ### Fixed - Fixed issue where the custom report "reset" button was not working as expected [#5649](https://github.com/ethyca/fides/pull/5649) diff --git a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts index b69cf28f20..aa5c1f4fe3 100644 --- a/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts +++ b/clients/admin-ui/src/features/privacy-requests/privacy-requests.slice.ts @@ -110,8 +110,12 @@ export const requestCSVDownload = async ({ }, }, ) - .then((response) => { + .then(async (response) => { if (!response.ok) { + if (response.status === 400) { + const errorData = await response.json(); + throw new Error(errorData.detail || "Bad request error"); + } throw new Error("Got a bad response from the server"); } return response.blob(); diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index bfe42a1455..0a39866924 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -672,6 +672,21 @@ def attach_resume_instructions(privacy_request: PrivacyRequest) -> None: ) +def _validate_result_size(query: Query) -> None: + """ + Validates the result size is less than maximum allowed by settings. + Raises an HTTPException if result size is greater than maximum. + Result size is determined by running an up-front "count" query. + """ + row_count = query.count() + max_rows = CONFIG.admin_ui.max_privacy_request_download_rows + if row_count > max_rows: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"Requested privacy request report would contain {row_count} rows. A maximum of {max_rows} rows is permitted. Please narrow your date range and try again.", + ) + + def _shared_privacy_request_search( *, db: Session, @@ -740,6 +755,7 @@ def _shared_privacy_request_search( query = _sort_privacy_request_queryset(query, sort_field, sort_direction) if download_csv: + _validate_result_size(query) # Returning here if download_csv param was specified logger.info("Downloading privacy requests as csv") return privacy_request_csv_download(db, query) diff --git a/src/fides/config/admin_ui_settings.py b/src/fides/config/admin_ui_settings.py index 67cacba456..cd50a01675 100644 --- a/src/fides/config/admin_ui_settings.py +++ b/src/fides/config/admin_ui_settings.py @@ -17,4 +17,8 @@ class AdminUISettings(FidesSettings): url: SerializeAsAny[Optional[AnyHttpUrlStringRemovesSlash]] = Field( default=None, description="The base URL for the Admin UI." ) + max_privacy_request_download_rows: int = Field( + default=100000, + description="The maximum number of rows permitted to be returned in a privacy request report download", + ) model_config = SettingsConfigDict(env_prefix="FIDES__ADMIN_UI__") diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index d030919aed..19075f09de 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -3498,6 +3498,14 @@ def allow_custom_privacy_request_fields_in_request_execution_disabled(): ) +@pytest.fixture(scope="function") +def set_max_privacy_request_download_rows(): + original_value = CONFIG.admin_ui.max_privacy_request_download_rows + CONFIG.admin_ui.max_privacy_request_download_rows = 1 + yield + CONFIG.admin_ui.max_privacy_request_download_rows = original_value + + @pytest.fixture(scope="function") def subject_request_download_ui_enabled(): original_value = CONFIG.security.subject_request_download_ui_enabled diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index c4f386d7b0..2cc83707b3 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -1951,6 +1951,19 @@ def test_get_privacy_requests_csv_format( privacy_request.delete(db) + def test_get_privacy_requests_csv_format_max_rows_limit( + self, + db, + generate_auth_header, + api_client, + url, + privacy_requests, + set_max_privacy_request_download_rows, + ): + auth_header = generate_auth_header(scopes=[PRIVACY_REQUEST_READ]) + response = api_client.get(url + f"?download_csv=True", headers=auth_header) + assert 400 == response.status_code + def test_get_requires_input_privacy_request_resume_info( self, db, privacy_request, generate_auth_header, api_client, url ):