From 29be6f7b585381df6d8ac09d13a644611b353695 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 9 Jul 2024 15:25:09 +0200 Subject: [PATCH 01/10] feat: send environment and version to Sentry --- udata/frontend/__init__.py | 2 +- udata/sentry.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/udata/frontend/__init__.py b/udata/frontend/__init__.py index 03544af926..662432d43c 100644 --- a/udata/frontend/__init__.py +++ b/udata/frontend/__init__.py @@ -23,7 +23,7 @@ @hook.app_template_global() -def package_version(name): +def package_version(name: str): return pkg_resources.get_distribution(name).version diff --git a/udata/sentry.py b/udata/sentry.py index b81e26b027..0bf46a4828 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -8,7 +8,9 @@ from udata import entrypoints from udata.core.storages.api import UploadProgress +from .app import UDataApp from .auth import PermissionDenied +from .frontend import package_version log = logging.getLogger(__name__) @@ -24,7 +26,7 @@ IGNORED_EXCEPTIONS = HTTPException, PermissionDenied, UploadProgress -def public_dsn(dsn): +def public_dsn(dsn: str): """Check if DSN is public or raise a warning and turn it into a public one""" m = RE_DSN.match(dsn) if not m: @@ -41,7 +43,7 @@ def public_dsn(dsn): return public -def init_app(app): +def init_app(app: UDataApp): if app.config["SENTRY_DSN"]: try: import sentry_sdk @@ -62,6 +64,16 @@ def init_app(app): dsn=app.config["SENTRY_PUBLIC_DSN"], integrations=[FlaskIntegration(), CeleryIntegration()], ignore_errors=list(exceptions), + release=f"udata@{package_version('udata')}", + environment=app.config["SITE_ID"], + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for performance monitoring. + # We recommend adjusting this value in production. + traces_sample_rate=1.0, + # Experimental profiling + _experiments={ + "profiles_sample_rate": 1.0, + }, ) # Set log level From dfa8c49eaf7a6bd6a7a5367eb42bb4bff06045c7 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Wed, 10 Jul 2024 19:07:16 +0200 Subject: [PATCH 02/10] feat: do not use same Sentry sample rate for debug and prod --- udata/sentry.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/udata/sentry.py b/udata/sentry.py index 0bf46a4828..b8721cb2f9 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -60,6 +60,10 @@ def init_app(app: UDataApp): for exception in IGNORED_EXCEPTIONS: exceptions.add(exception) + sentry_sample_rate: float = 0.1 + if app.config.get("DEBUG"): + sentry_sample_rate = 1.0 + sentry_sdk.init( dsn=app.config["SENTRY_PUBLIC_DSN"], integrations=[FlaskIntegration(), CeleryIntegration()], @@ -68,11 +72,11 @@ def init_app(app: UDataApp): environment=app.config["SITE_ID"], # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. - # We recommend adjusting this value in production. - traces_sample_rate=1.0, + # Sentry recommends adjusting this value in production. + traces_sample_rate=sentry_sample_rate, # Experimental profiling _experiments={ - "profiles_sample_rate": 1.0, + "profiles_sample_rate": sentry_sample_rate, }, ) From 11be0f6446e8cbccaa981d40d76123dfe4d50745 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 12 Jul 2024 14:31:48 +0200 Subject: [PATCH 03/10] fix: use safer dict getters --- udata/sentry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/udata/sentry.py b/udata/sentry.py index b8721cb2f9..82b7ab2d1a 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -61,7 +61,7 @@ def init_app(app: UDataApp): exceptions.add(exception) sentry_sample_rate: float = 0.1 - if app.config.get("DEBUG"): + if app.config.get("DEBUG", None): sentry_sample_rate = 1.0 sentry_sdk.init( @@ -69,7 +69,7 @@ def init_app(app: UDataApp): integrations=[FlaskIntegration(), CeleryIntegration()], ignore_errors=list(exceptions), release=f"udata@{package_version('udata')}", - environment=app.config["SITE_ID"], + environment=app.config.get("SITE_ID", None), # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # Sentry recommends adjusting this value in production. From 6f036c53651cc80bf12d6571acc0968e35fc365c Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 12 Jul 2024 11:20:07 +0200 Subject: [PATCH 04/10] chore: add sentry-sdk[flask] to install dependencies, and fix dependencies incompabilities --- requirements/install.in | 1 + requirements/install.pip | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements/install.in b/requirements/install.in index 132f959283..750d347687 100644 --- a/requirements/install.in +++ b/requirements/install.in @@ -42,6 +42,7 @@ pytz==2024.1 redis==4.5.2 rdflib==6.0.0 requests==2.24.0 +sentry-sdk[flask]==2.9.0 speaklater==1.3 StringDist==1.0.9 tlds diff --git a/requirements/install.pip b/requirements/install.pip index 115448cfe2..e6871c6ae4 100644 --- a/requirements/install.pip +++ b/requirements/install.pip @@ -245,7 +245,7 @@ referencing==0.35.1 # jsonschema-specifications regex==2024.5.15 # via awesome-slugify -requests==2.24.0 +requests==2.32.3 # via -r requirements/install.in rpds-py==0.19.0 # via @@ -253,6 +253,8 @@ rpds-py==0.19.0 # referencing s3transfer==0.6.2 # via boto3 +sentry-sdk==2.9.0 + # via -r requirements/install.in six==1.16.0 # via # bcrypt @@ -279,7 +281,7 @@ uritools==4.0.3 # via urlextract urlextract==0.14.0 # via -r requirements/install.in -urllib3==1.25.11 +urllib3==1.26.19 # via # botocore # requests From d3368c471f0d11120e25ede460fc9fdd333e7491 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 12 Jul 2024 14:28:06 +0200 Subject: [PATCH 05/10] style: add type hints --- udata/frontend/__init__.py | 2 +- udata/sentry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/udata/frontend/__init__.py b/udata/frontend/__init__.py index 662432d43c..7216b8921e 100644 --- a/udata/frontend/__init__.py +++ b/udata/frontend/__init__.py @@ -23,7 +23,7 @@ @hook.app_template_global() -def package_version(name: str): +def package_version(name: str) -> str: return pkg_resources.get_distribution(name).version diff --git a/udata/sentry.py b/udata/sentry.py index 82b7ab2d1a..8d7a69ce64 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -26,7 +26,7 @@ IGNORED_EXCEPTIONS = HTTPException, PermissionDenied, UploadProgress -def public_dsn(dsn: str): +def public_dsn(dsn: str) -> str | None: """Check if DSN is public or raise a warning and turn it into a public one""" m = RE_DSN.match(dsn) if not m: From d011a968836d7a3101d79e719dbc683d1a36058f Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 15:25:58 +0200 Subject: [PATCH 06/10] chore: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3b3548c5..84861fc256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Current (in progress) +- Adds latest `sentry-sdk[flask]` as an install dependency, and update Sentry logic to be able to send environment, app version and profiling/performance info [#3086](https://github.com/opendatateam/udata/pull/3086) - Add a `archived` field for reuses [#3088](https://github.com/opendatateam/udata/pull/3088) - Add linter and formatter with `pyproject.toml` config, add lint and formatting step in CI, add pre-commit hook to lint and format, update docs and lint and format the code [#3085](https://github.com/opendatateam/udata/pull/3085) - Revamp CORS: remove flask-cors to always returns 204 on OPTIONS request [#3074](https://github.com/opendatateam/udata/pull/3074) From c3a729394148a8e26bd82d62c9140162f90bf788 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 15:30:32 +0200 Subject: [PATCH 07/10] feat: sentry sample rate as app config --- udata/sentry.py | 4 ++-- udata/settings.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/udata/sentry.py b/udata/sentry.py index 8d7a69ce64..a10d8faf73 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -73,10 +73,10 @@ def init_app(app: UDataApp): # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # Sentry recommends adjusting this value in production. - traces_sample_rate=sentry_sample_rate, + traces_sample_rate=app.config.get("SENTRY_SAMPLE_RATE", None), # Experimental profiling _experiments={ - "profiles_sample_rate": sentry_sample_rate, + "profiles_sample_rate": app.config.get("SENTRY_SAMPLE_RATE", None), }, ) diff --git a/udata/settings.py b/udata/settings.py index e426e1212a..9b3815821b 100644 --- a/udata/settings.py +++ b/udata/settings.py @@ -113,6 +113,7 @@ class Defaults(object): SENTRY_USER_ATTRS = ["slug", "email", "fullname"] SENTRY_LOGGING = "WARNING" SENTRY_IGNORE_EXCEPTIONS = [] + SENTRY_SAMPLE_RATE: float = 1.0 # Flask WTF settings CSRF_SESSION_KEY = "Default uData csrf key" From 15c9df08e7b823c2d79b568250b4dcc227555e25 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 16 Jul 2024 15:07:43 +0200 Subject: [PATCH 08/10] fix: remove useless code --- udata/sentry.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/udata/sentry.py b/udata/sentry.py index a10d8faf73..621e60eab2 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -60,10 +60,6 @@ def init_app(app: UDataApp): for exception in IGNORED_EXCEPTIONS: exceptions.add(exception) - sentry_sample_rate: float = 0.1 - if app.config.get("DEBUG", None): - sentry_sample_rate = 1.0 - sentry_sdk.init( dsn=app.config["SENTRY_PUBLIC_DSN"], integrations=[FlaskIntegration(), CeleryIntegration()], From 96eb0926fe45b2e7513f3186014fc3d1ce4dad46 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 16 Jul 2024 15:22:29 +0200 Subject: [PATCH 09/10] clean: simplify sentry config --- udata/sentry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/udata/sentry.py b/udata/sentry.py index 621e60eab2..d372bb45df 100644 --- a/udata/sentry.py +++ b/udata/sentry.py @@ -70,10 +70,7 @@ def init_app(app: UDataApp): # of transactions for performance monitoring. # Sentry recommends adjusting this value in production. traces_sample_rate=app.config.get("SENTRY_SAMPLE_RATE", None), - # Experimental profiling - _experiments={ - "profiles_sample_rate": app.config.get("SENTRY_SAMPLE_RATE", None), - }, + profiles_sample_rate=app.config.get("SENTRY_SAMPLE_RATE", None), ) # Set log level From 6afe805b3c395ba72469c82a61efce18d8ee7c78 Mon Sep 17 00:00:00 2001 From: maudetes Date: Wed, 24 Jul 2024 12:20:24 +0200 Subject: [PATCH 10/10] Upgrade pip-tools rev in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fbb95c8fe..566f1e57a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/jazzband/pip-tools - rev: 6.2.0 + rev: 7.4.1 hooks: - id: pip-compile name: pip-compile install.in