feat: Add datadog_diagnostics plugin app

See #692

Testing setup:
https://2u-internal.atlassian.net/wiki/spaces/ENG/pages/1173618788/Running+Datadog+in+devstack

And then in lms-shell:

```
make requirements
pip install ddtrace
pip install -e /edx/src/archexp/
./wrap-datadog.sh ./server.sh
```

Expect to see this log message:
`Attached MissingSpanProccessor for Datadog diagnostics`

CHANGELOG.rst

@@ -14,6 +14,12 @@ Change Log
 Unreleased
 ~~~~~~~~~~
 
+[3.4.0] - 2024-07-10
+~~~~~~~~~~~~~~~~~~~~
+Added
+-----
+* Added ``datadog_diagnostics`` plugin app
+
 [3.3.2] - 2024-04-19
 ~~~~~~~~~~~~~~~~~~~~
 Changed

edx_arch_experiments/__init__.py

@@ -2,4 +2,4 @@
 A plugin to include applications under development by the architecture team at 2U.
 """
 
-__version__ = '3.3.2'
+__version__ = '3.4.0'

edx_arch_experiments/datadog_diagnostics/README.rst

@@ -0,0 +1,13 @@
+Datadog Diagnostics
+###################
+
+When installed in the LMS as a plugin app, the ``datadog_diagnostics`` app adds additional logging for debugging our Datadog integration.
+
+This is intended as a temporary situation while we debug the `trace concatenation issue <https://github.com/edx/edx-arch-experiments/issues/692>`_.
+
+Usage
+*****
+
+In LMS:
+
+- Install ``edx-arch-experiments`` as a dependency

edx_arch_experiments/datadog_diagnostics/apps.py

@@ -0,0 +1,50 @@
+"""
+App for emitting additional diagnostic information for the Datadog integration.
+"""
+
+import logging
+
+from django.apps import AppConfig
+
+log = logging.getLogger(__name__)
+
+
+class MissingSpanProccessor:
+    """Datadog span processor that logs unfinished spans at shutdown."""
+    spans_started = 0
+    spans_finished = 0
+    open_spans = {}
+
+    def on_span_start(self, span):
+        self.spans_started += 1
+        self.open_spans[span.span_id] = span
+
+    def on_span_finish(self, span):
+        self.spans_finished += 1
+        del self.open_spans[span.span_id]
+
+    def shutdown(self, _timeout):
+        log.info(f"Spans created = {self.spans_started}; spans finished = {self.spans_finished}")
+        for span in self.open_spans.values():
+            log.error(f"Span created but not finished: {span._pprint()}")  # pylint: disable=protected-access
+
+
+class DatadogDiagnostics(AppConfig):
+    """
+    Django application to log diagnostic information for Datadog.
+    """
+    name = 'edx_arch_experiments.datadog_diagnostics'
+
+    # Mark this as a plugin app
+    plugin_app = {}
+
+    def ready(self):
+        try:
+            from ddtrace import tracer  # pylint: disable=import-outside-toplevel
+            tracer._span_processors.append(MissingSpanProccessor())  # pylint: disable=protected-access
+            log.info("Attached MissingSpanProccessor for Datadog diagnostics")
+        except ImportError:
+            log.warning(
+                "Unable to attach MissingSpanProccessor for Datadog diagnostics"
+                " -- ddtrace module not found."
+            )

setup.py

@@ -164,6 +164,7 @@ def is_requirement(line):
             "arch_experiments = edx_arch_experiments.apps:EdxArchExperimentsConfig",
             "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher",
             "codejail_service = edx_arch_experiments.codejail_service.apps:CodejailService",
+            "datadog_diagnostics = edx_arch_experiments.datadog_diagnostics.apps:DatadogDiagnostics",
         ],
         "cms.djangoapp": [
             "config_watcher = edx_arch_experiments.config_watcher.apps:ConfigWatcher",