Use OTel for performance instrumentation (PoC)

sentry_sdk/client.py

@@ -27,7 +27,7 @@
     VERSION,
     ClientConstructor,
 )
-from sentry_sdk.integrations import setup_integrations
+from sentry_sdk.integrations import _DEFAULT_INTEGRATIONS, setup_integrations
 from sentry_sdk.utils import ContextVar
 from sentry_sdk.sessions import SessionFlusher
 from sentry_sdk.envelope import Envelope
@@ -237,6 +237,15 @@ def _capture_envelope(envelope):
                     )
                 )
 
+            if self.options["_experiments"].get("otel_powered_performance", False):
+                logger.debug(
+                    "[OTel] Enabling experimental OTel-powered performance monitoring."
+                )
+                self.options["instrumenter"] = INSTRUMENTER.OTEL
+                _DEFAULT_INTEGRATIONS.append(
+                    "sentry_sdk.integrations.opentelemetry.OpenTelemetryIntegration",
+                )
+
             self.integrations = setup_integrations(
                 self.options["integrations"],
                 with_defaults=self.options["default_integrations"],

sentry_sdk/consts.py

@@ -39,6 +39,7 @@
             # TODO: Remove these 2 profiling related experiments
             "profiles_sample_rate": Optional[float],
             "profiler_mode": Optional[ProfilerMode],
+            "otel_powered_performance": Optional[bool],
         },
         total=False,
     )

sentry_sdk/integrations/__init__.py

@@ -1,29 +1,29 @@
-"""This package"""
 from __future__ import absolute_import
-
 from threading import Lock
 
 from sentry_sdk._compat import iteritems
+from sentry_sdk._types import TYPE_CHECKING
 from sentry_sdk.utils import logger
 
-from sentry_sdk._types import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Callable
     from typing import Dict
     from typing import Iterator
     from typing import List
     from typing import Set
-    from typing import Tuple
     from typing import Type
 
 
 _installer_lock = Lock()
 _installed_integrations = set()  # type: Set[str]
 
 
-def _generate_default_integrations_iterator(integrations, auto_enabling_integrations):
-    # type: (Tuple[str, ...], Tuple[str, ...]) -> Callable[[bool], Iterator[Type[Integration]]]
+def _generate_default_integrations_iterator(
+    integrations,  # type: List[str]
+    auto_enabling_integrations,  # type: List[str]
+):
+    # type: (...) -> Callable[[bool], Iterator[Type[Integration]]]
 
     def iter_default_integrations(with_auto_enabling_integrations):
         # type: (bool) -> Iterator[Type[Integration]]
@@ -51,38 +51,40 @@ def iter_default_integrations(with_auto_enabling_integrations):
     return iter_default_integrations
 
 
-_AUTO_ENABLING_INTEGRATIONS = (
-    "sentry_sdk.integrations.django.DjangoIntegration",
-    "sentry_sdk.integrations.flask.FlaskIntegration",
-    "sentry_sdk.integrations.starlette.StarletteIntegration",
-    "sentry_sdk.integrations.fastapi.FastApiIntegration",
+_DEFAULT_INTEGRATIONS = [
+    # stdlib/base runtime integrations
+    "sentry_sdk.integrations.argv.ArgvIntegration",
+    "sentry_sdk.integrations.atexit.AtexitIntegration",
+    "sentry_sdk.integrations.dedupe.DedupeIntegration",
+    "sentry_sdk.integrations.excepthook.ExcepthookIntegration",
+    "sentry_sdk.integrations.logging.LoggingIntegration",
+    "sentry_sdk.integrations.modules.ModulesIntegration",
+    "sentry_sdk.integrations.stdlib.StdlibIntegration",
+    "sentry_sdk.integrations.threading.ThreadingIntegration",
+]
+
+_AUTO_ENABLING_INTEGRATIONS = [
+    "sentry_sdk.integrations.aiohttp.AioHttpIntegration",
+    "sentry_sdk.integrations.boto3.Boto3Integration",
     "sentry_sdk.integrations.bottle.BottleIntegration",
-    "sentry_sdk.integrations.falcon.FalconIntegration",
-    "sentry_sdk.integrations.sanic.SanicIntegration",
     "sentry_sdk.integrations.celery.CeleryIntegration",
+    "sentry_sdk.integrations.django.DjangoIntegration",
+    "sentry_sdk.integrations.falcon.FalconIntegration",
+    "sentry_sdk.integrations.fastapi.FastApiIntegration",
+    "sentry_sdk.integrations.flask.FlaskIntegration",
+    "sentry_sdk.integrations.httpx.HttpxIntegration",
+    "sentry_sdk.integrations.pyramid.PyramidIntegration",
+    "sentry_sdk.integrations.redis.RedisIntegration",
     "sentry_sdk.integrations.rq.RqIntegration",
-    "sentry_sdk.integrations.aiohttp.AioHttpIntegration",
-    "sentry_sdk.integrations.tornado.TornadoIntegration",
+    "sentry_sdk.integrations.sanic.SanicIntegration",
     "sentry_sdk.integrations.sqlalchemy.SqlalchemyIntegration",
-    "sentry_sdk.integrations.redis.RedisIntegration",
-    "sentry_sdk.integrations.pyramid.PyramidIntegration",
-    "sentry_sdk.integrations.boto3.Boto3Integration",
-    "sentry_sdk.integrations.httpx.HttpxIntegration",
-)
+    "sentry_sdk.integrations.starlette.StarletteIntegration",
+    "sentry_sdk.integrations.tornado.TornadoIntegration",
+]
 
 
 iter_default_integrations = _generate_default_integrations_iterator(
-    integrations=(
-        # stdlib/base runtime integrations
-        "sentry_sdk.integrations.logging.LoggingIntegration",
-        "sentry_sdk.integrations.stdlib.StdlibIntegration",
-        "sentry_sdk.integrations.excepthook.ExcepthookIntegration",
-        "sentry_sdk.integrations.dedupe.DedupeIntegration",
-        "sentry_sdk.integrations.atexit.AtexitIntegration",
-        "sentry_sdk.integrations.modules.ModulesIntegration",
-        "sentry_sdk.integrations.argv.ArgvIntegration",
-        "sentry_sdk.integrations.threading.ThreadingIntegration",
-    ),
+    integrations=_DEFAULT_INTEGRATIONS,
     auto_enabling_integrations=_AUTO_ENABLING_INTEGRATIONS,
 )
 
@@ -93,8 +95,10 @@ def setup_integrations(
     integrations, with_defaults=True, with_auto_enabling_integrations=False
 ):
     # type: (List[Integration], bool, bool) -> Dict[str, Integration]
-    """Given a list of integration instances this installs them all.  When
-    `with_defaults` is set to `True` then all default integrations are added
+    """
+    Given a list of integration instances, this installs them all.
+
+    When `with_defaults` is set to `True` all default integrations are added
     unless they were already provided before.
     """
     integrations = dict(

sentry_sdk/integrations/opentelemetry/__init__.py

@@ -1,3 +1,7 @@
+from sentry_sdk.integrations.opentelemetry.integration import (  # noqa: F401
+    OpenTelemetryIntegration,
+)
+
 from sentry_sdk.integrations.opentelemetry.span_processor import (  # noqa: F401
     SentrySpanProcessor,
 )

sentry_sdk/integrations/opentelemetry/integration.py

@@ -0,0 +1,163 @@
+"""
+IMPORTANT: The contents of this file are part of a proof of concept and as such
+are experimental and not suitable for production use. They may be changed or
+removed at any time without prior notice.
+"""
+import sys
+from importlib import import_module
+
+from sentry_sdk.integrations import DidNotEnable, Integration
+from sentry_sdk.integrations.opentelemetry.span_processor import SentrySpanProcessor
+from sentry_sdk.integrations.opentelemetry.propagator import SentryPropagator
+from sentry_sdk.integrations.modules import _get_installed_modules
+from sentry_sdk.utils import logger
+from sentry_sdk._types import TYPE_CHECKING
+
+try:
+    from opentelemetry import trace  # type: ignore
+    from opentelemetry.instrumentation.auto_instrumentation._load import (  # type: ignore
+        _load_distro,
+        _load_instrumentors,
+    )
+    from opentelemetry.propagate import set_global_textmap  # type: ignore
+    from opentelemetry.sdk.trace import TracerProvider  # type: ignore
+except ImportError:
+    raise DidNotEnable("opentelemetry not installed")
+
+if TYPE_CHECKING:
+    from typing import Dict
+
+
+INSTRUMENTED_CLASSES = {
+    # A mapping of packages to (original class, instrumented class) pairs. This
+    # is used to post-instrument any classes that were imported before OTel
+    # instrumentation took place.
+    "fastapi": (
+        "fastapi.FastAPI",
+        "opentelemetry.instrumentation.fastapi._InstrumentedFastAPI",
+    ),
+    "flask": ("flask.Flask", "opentelemetry.instrumentation.flask._InstrumentedFlask"),
+}
+
+
+class OpenTelemetryIntegration(Integration):
+    identifier = "opentelemetry"
+
+    @staticmethod
+    def setup_once():
+        # type: () -> None
+        logger.warning(
+            "[OTel] Initializing highly experimental OpenTelemetry support. Use at your own risk."
+        )
+
+        original_classes = _record_unpatched_classes()
+
+        try:
+            distro = _load_distro()
+            distro.configure()
+            _load_instrumentors(distro)
+        except Exception:
+            logger.exception("[OTel] Failed to auto-initialize OpenTelemetry")
+
+        _patch_remaining_classes(original_classes)
+        _setup_sentry_tracing()
+
+        logger.debug("[OTel] Finished setting up OpenTelemetry integration")
+
+
+def _record_unpatched_classes():
+    # type: () -> Dict[str, type]
+    """
+    Keep references to classes that are about to be instrumented.
+
+    Used to search for unpatched classes after the instrumentation has run so
+    that they can be patched manually.
+    """
+    installed_packages = _get_installed_modules()
+
+    original_classes = {}
+
+    for package, (orig_path, _) in INSTRUMENTED_CLASSES.items():
+        if package in installed_packages:
+            try:
+                original_cls = _import_by_path(orig_path)
+            except (AttributeError, ImportError):
+                logger.debug("[OTel] Failed to import %s", orig_path)
+                continue
+
+            original_classes[package] = original_cls
+
+    return original_classes
+
+
+def _patch_remaining_classes(original_classes):
+    # type: (Dict[str, type]) -> None
+    """
+    Best-effort attempt to patch any uninstrumented classes in sys.modules.
+
+    This enables us to not care about the order of imports and sentry_sdk.init()
+    in user code. If e.g. the Flask class had been imported before sentry_sdk
+    was init()ed (and therefore before the OTel instrumentation ran), it would
+    not be instrumented. This function goes over remaining uninstrumented
+    occurrences of the class in sys.modules and patches them.
+
+    Since this is looking for exact matches, it will not work in some scenarios
+    (e.g. if someone is not using the specific class explicitly, but rather
+    inheriting from it). In those cases it's still necessary to sentry_sdk.init()
+    before importing anything that's supposed to be instrumented.
+    """
+    for module_name, module in sys.modules.copy().items():
+        if (
+            module_name.startswith("sentry_sdk")
+            or module_name in sys.builtin_module_names
+        ):
+            continue
+
+        for package, original_cls in original_classes.items():
+            original_path, instrumented_path = INSTRUMENTED_CLASSES[package]
+
+            try:
+                cls = _import_by_path(original_path)
+            except (AttributeError, ImportError):
+                logger.debug(
+                    "[OTel] Failed to check if class has been instrumented: %s",
+                    original_path,
+                )
+                continue
+
+            if not cls.__module__.startswith("opentelemetry."):
+                # the class wasn't instrumented, don't do any additional patching
+                continue
+
+            for var_name, var in vars(module).copy().items():
+                if var == original_cls:
+                    logger.debug(
+                        "[OTel] Additionally patching %s from %s with %s",
+                        original_cls,
+                        module_name,
+                        instrumented_path,
+                    )
+
+                    try:
+                        isntrumented_cls = _import_by_path(instrumented_path)
+                    except (AttributeError, ImportError):
+                        logger.debug("[OTel] Failed to import %s", instrumented_path)
+
+                    setattr(module, var_name, isntrumented_cls)
+
+
+def _import_by_path(path):
+    # type: (str) -> type
+    parts = path.rsplit(".", maxsplit=1)
+    return getattr(import_module(parts[0]), parts[-1])
+
+
+def _setup_sentry_tracing():
+    # type: () -> None
+    provider = TracerProvider()
+
+    provider.add_span_processor(SentrySpanProcessor())
+
+    trace.set_tracer_provider(provider)
+
+    set_global_textmap(SentryPropagator())

sentry_sdk/integrations/opentelemetry/span_processor.py

@@ -169,6 +169,7 @@ def on_end(self, otel_span):
             sentry_span.set_context(
                 OPEN_TELEMETRY_CONTEXT, self._get_otel_context(otel_span)
             )
+            self._update_transaction_with_otel_data(sentry_span, otel_span)
 
         else:
             self._update_span_with_otel_data(sentry_span, otel_span)
@@ -306,3 +307,21 @@ def _update_span_with_otel_data(self, sentry_span, otel_span):
 
         sentry_span.op = op
         sentry_span.description = description
+
+    def _update_transaction_with_otel_data(self, sentry_span, otel_span):
+        # type: (SentrySpan, OTelSpan) -> None
+        http_method = otel_span.attributes.get(SpanAttributes.HTTP_METHOD)
+
+        if http_method:
+            status_code = otel_span.attributes.get(SpanAttributes.HTTP_STATUS_CODE)
+            if status_code:
+                sentry_span.set_http_status(status_code)
+
+            op = "http"
+
+            if otel_span.kind == SpanKind.SERVER:
+                op += ".server"
+            elif otel_span.kind == SpanKind.CLIENT:
+                op += ".client"
+
+            sentry_span.op = op