From cb26cc734c48d5e332076e745594c3dd6e04f9be Mon Sep 17 00:00:00 2001
From: Tim McCormack <tmccormack@edx.org>
Date: Wed, 10 Jul 2024 15:52:06 -0400
Subject: [PATCH] feat: Add datadog_diagnostics plugin app (#722)

See https://github.com/edx/edx-arch-experiments/issues/692

Testing setup:
https://2u-internal.atlassian.net/wiki/spaces/ENG/pages/1173618788/Running+Datadog+in+devstack

And then in lms-shell:

```
make requirements
pip install ddtrace
pip install -e /edx/src/archexp/
./wrap-datadog.sh ./server.sh
```

Expect to see this log message:
`Attached MissingSpanProccessor for Datadog diagnostics`

NOTE: This prints "Spans created = 0; spans finished = 0" in devstack when
shut down with ctrl-c, but not when restarted due to autoreload (where it
prints correct info). Something is initializing Django twice and one span
processor is getting span info while the other is printing at shutdown.
There's more to debug here, but it seems stable enough to least try
deploying it.
---
 CHANGELOG.rst                                 |  6 +++
 edx_arch_experiments/__init__.py              |  2 +-
 .../datadog_diagnostics/README.rst            | 13 +++++
 .../datadog_diagnostics/__init__.py           |  0
 .../datadog_diagnostics/apps.py               | 50 +++++++++++++++++++
 setup.py                                      |  1 +
 6 files changed, 71 insertions(+), 1 deletion(-)
 create mode 100644 edx_arch_experiments/datadog_diagnostics/README.rst
 create mode 100644 edx_arch_experiments/datadog_diagnostics/__init__.py
 create mode 100644 edx_arch_experiments/datadog_diagnostics/apps.py

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 3429ebb..16ee9ed 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -14,6 +14,12 @@ Change Log
 Unreleased
 ~~~~~~~~~~
 
+[3.4.0] - 2024-07-10
+~~~~~~~~~~~~~~~~~~~~
+Added
+-----
+* Added ``datadog_diagnostics`` plugin app
+
 [3.3.2] - 2024-04-19
 ~~~~~~~~~~~~~~~~~~~~
 Changed
diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py
index bdea0fa..37183df 100644
--- a/edx_arch_experiments/__init__.py
+++ b/edx_arch_experiments/__init__.py
@@ -2,4 +2,4 @@
 A plugin to include applications under development by the architecture team at 2U.
 """
 
-__version__ = '3.3.2'
+__version__ = '3.4.0'
diff --git a/edx_arch_experiments/datadog_diagnostics/README.rst b/edx_arch_experiments/datadog_diagnostics/README.rst
new file mode 100644
index 0000000..c51dd6d
--- /dev/null
+++ b/edx_arch_experiments/datadog_diagnostics/README.rst
@@ -0,0 +1,13 @@
+Datadog Diagnostics
+###################
+
+When installed in the LMS as a plugin app, the ``datadog_diagnostics`` app adds additional logging for debugging our Datadog integration.
+
+This is intended as a temporary situation while we debug the `trace concatenation issue <https://github.com/edx/edx-arch-experiments/issues/692>`_.
+
+Usage
+*****
+
+In LMS:
+
+- Install ``edx-arch-experiments`` as a dependency
diff --git a/edx_arch_experiments/datadog_diagnostics/__init__.py b/edx_arch_experiments/datadog_diagnostics/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py
new file mode 100644
index 0000000..86494a8
--- /dev/null
+++ b/edx_arch_experiments/datadog_diagnostics/apps.py
@@ -0,0 +1,50 @@
+"""
+App for emitting additional diagnostic information for the Datadog integration.
+"""
+
+import logging
+
+from django.apps import AppConfig
+
+log = logging.getLogger(__name__)
+
+
+class MissingSpanProcessor:
+    """Datadog span processor that logs unfinished spans at shutdown."""
+    spans_started = 0
+    spans_finished = 0
+    open_spans = {}
+
+    def on_span_start(self, span):
+        self.spans_started += 1
+        self.open_spans[span.span_id] = span
+
+    def on_span_finish(self, span):
+        self.spans_finished += 1
+        del self.open_spans[span.span_id]
+
+    def shutdown(self, _timeout):
+        log.info(f"Spans created = {self.spans_started}; spans finished = {self.spans_finished}")
+        for span in self.open_spans.values():
+            log.error(f"Span created but not finished: {span._pprint()}")  # pylint: disable=protected-access
+
+
+class DatadogDiagnostics(AppConfig):
+    """
+    Django application to log diagnostic information for Datadog.
+    """
+    name = 'edx_arch_experiments.datadog_diagnostics'
+
+    # Mark this as a plugin app
+    plugin_app = {}
+
+    def ready(self):
+        try:
+            from ddtrace import tracer  # pylint: disable=import-outside-toplevel
+            tracer._span_processors.append(MissingSpanProcessor())  # pylint: disable=protected-access
+            log.info("Attached MissingSpanProcessor for Datadog diagnostics")
+        except ImportError:
+            log.warning(
+                "Unable to attach MissingSpanProcessor for Datadog diagnostics"
+                " -- ddtrace module not found."
+            )
diff --git a/setup.py b/setup.py
index a7ee121..94a6eee 100644
--- a/setup.py
+++ b/setup.py
@@ -164,6 +164,7 @@ def is_requirement(line):
             "arch_experiments = edx_arch_experiments.apps:EdxArchExperimentsConfig",
             "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher",
             "codejail_service = edx_arch_experiments.codejail_service.apps:CodejailService",
+            "datadog_diagnostics = edx_arch_experiments.datadog_diagnostics.apps:DatadogDiagnostics",
         ],
         "cms.djangoapp": [
             "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher",