From 8e4828c8a32572a296056d104627d18b267d602f Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 10 Jul 2024 15:22:35 +0000 Subject: [PATCH 1/2] feat: Add datadog_diagnostics plugin app See https://github.com/edx/edx-arch-experiments/issues/692 Testing setup: https://2u-internal.atlassian.net/wiki/spaces/ENG/pages/1173618788/Running+Datadog+in+devstack And then in lms-shell: ``` make requirements pip install ddtrace pip install -e /edx/src/archexp/ ./wrap-datadog.sh ./server.sh ``` Expect to see this log message: `Attached MissingSpanProccessor for Datadog diagnostics` NOTE: This prints "Spans created = 0; spans finished = 0" in devstack when shut down with ctrl-c, but not when restarted due to autoreload (where it prints correct info). Something is initializing Django twice and one span processor is getting span info while the other is printing at shutdown. There's more to debug here, but it seems stable enough to least try deploying it. --- CHANGELOG.rst | 6 +++ edx_arch_experiments/__init__.py | 2 +- .../datadog_diagnostics/README.rst | 13 +++++ .../datadog_diagnostics/__init__.py | 0 .../datadog_diagnostics/apps.py | 50 +++++++++++++++++++ setup.py | 1 + 6 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 edx_arch_experiments/datadog_diagnostics/README.rst create mode 100644 edx_arch_experiments/datadog_diagnostics/__init__.py create mode 100644 edx_arch_experiments/datadog_diagnostics/apps.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3429ebb..16ee9ed 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,12 @@ Change Log Unreleased ~~~~~~~~~~ +[3.4.0] - 2024-07-10 +~~~~~~~~~~~~~~~~~~~~ +Added +----- +* Added ``datadog_diagnostics`` plugin app + [3.3.2] - 2024-04-19 ~~~~~~~~~~~~~~~~~~~~ Changed diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py index bdea0fa..37183df 100644 --- a/edx_arch_experiments/__init__.py +++ b/edx_arch_experiments/__init__.py @@ -2,4 +2,4 @@ A plugin to include applications under development by the architecture team at 2U. """ -__version__ = '3.3.2' +__version__ = '3.4.0' diff --git a/edx_arch_experiments/datadog_diagnostics/README.rst b/edx_arch_experiments/datadog_diagnostics/README.rst new file mode 100644 index 0000000..c51dd6d --- /dev/null +++ b/edx_arch_experiments/datadog_diagnostics/README.rst @@ -0,0 +1,13 @@ +Datadog Diagnostics +################### + +When installed in the LMS as a plugin app, the ``datadog_diagnostics`` app adds additional logging for debugging our Datadog integration. + +This is intended as a temporary situation while we debug the `trace concatenation issue `_. + +Usage +***** + +In LMS: + +- Install ``edx-arch-experiments`` as a dependency diff --git a/edx_arch_experiments/datadog_diagnostics/__init__.py b/edx_arch_experiments/datadog_diagnostics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py new file mode 100644 index 0000000..23b5e30 --- /dev/null +++ b/edx_arch_experiments/datadog_diagnostics/apps.py @@ -0,0 +1,50 @@ +""" +App for emitting additional diagnostic information for the Datadog integration. +""" + +import logging + +from django.apps import AppConfig + +log = logging.getLogger(__name__) + + +class MissingSpanProccessor: + """Datadog span processor that logs unfinished spans at shutdown.""" + spans_started = 0 + spans_finished = 0 + open_spans = {} + + def on_span_start(self, span): + self.spans_started += 1 + self.open_spans[span.span_id] = span + + def on_span_finish(self, span): + self.spans_finished += 1 + del self.open_spans[span.span_id] + + def shutdown(self, _timeout): + log.info(f"Spans created = {self.spans_started}; spans finished = {self.spans_finished}") + for span in self.open_spans.values(): + log.error(f"Span created but not finished: {span._pprint()}") # pylint: disable=protected-access + + +class DatadogDiagnostics(AppConfig): + """ + Django application to log diagnostic information for Datadog. + """ + name = 'edx_arch_experiments.datadog_diagnostics' + + # Mark this as a plugin app + plugin_app = {} + + def ready(self): + try: + from ddtrace import tracer # pylint: disable=import-outside-toplevel + tracer._span_processors.append(MissingSpanProccessor()) # pylint: disable=protected-access + log.info("Attached MissingSpanProccessor for Datadog diagnostics") + except ImportError: + log.warning( + "Unable to attach MissingSpanProccessor for Datadog diagnostics" + " -- ddtrace module not found." + ) diff --git a/setup.py b/setup.py index a7ee121..94a6eee 100644 --- a/setup.py +++ b/setup.py @@ -164,6 +164,7 @@ def is_requirement(line): "arch_experiments = edx_arch_experiments.apps:EdxArchExperimentsConfig", "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher", "codejail_service = edx_arch_experiments.codejail_service.apps:CodejailService", + "datadog_diagnostics = edx_arch_experiments.datadog_diagnostics.apps:DatadogDiagnostics", ], "cms.djangoapp": [ "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher", From 58b1e27e267b9fccb0a7a6ec1cc60579722fe4e7 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 10 Jul 2024 17:50:11 +0000 Subject: [PATCH 2/2] fixup! typo in class name --- edx_arch_experiments/datadog_diagnostics/apps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py index 23b5e30..86494a8 100644 --- a/edx_arch_experiments/datadog_diagnostics/apps.py +++ b/edx_arch_experiments/datadog_diagnostics/apps.py @@ -9,7 +9,7 @@ log = logging.getLogger(__name__) -class MissingSpanProccessor: +class MissingSpanProcessor: """Datadog span processor that logs unfinished spans at shutdown.""" spans_started = 0 spans_finished = 0 @@ -41,10 +41,10 @@ class DatadogDiagnostics(AppConfig): def ready(self): try: from ddtrace import tracer # pylint: disable=import-outside-toplevel - tracer._span_processors.append(MissingSpanProccessor()) # pylint: disable=protected-access - log.info("Attached MissingSpanProccessor for Datadog diagnostics") + tracer._span_processors.append(MissingSpanProcessor()) # pylint: disable=protected-access + log.info("Attached MissingSpanProcessor for Datadog diagnostics") except ImportError: log.warning( - "Unable to attach MissingSpanProccessor for Datadog diagnostics" + "Unable to attach MissingSpanProcessor for Datadog diagnostics" " -- ddtrace module not found." )