analytics(toolbar): record api requests in custom middleware (#78778)

Closes getsentry/sentry-toolbar#33

We're ready to release a developer toolbar for installation on 3rd party
sites. Once installed, toolbars will query sentry backend for data - see
the [repo](https://github.com/getsentry/sentry-toolbar) or /devtoolbar
folder of sentry for examples. This helps us get analytics on every
endpoint, whenever the `querryReferrer=devtoolbar` header is included.

src/sentry/api/analytics.py

@@ -33,6 +33,25 @@ class GroupSimilarIssuesEmbeddingsCountEvent(analytics.Event):
     )
 
 
+class DevToolbarApiRequestEvent(analytics.Event):
+    type = "devtoolbar.api_request"
+
+    attributes = (
+        analytics.Attribute("view_name"),
+        analytics.Attribute("route"),
+        analytics.Attribute("query_string"),
+        analytics.Attribute("origin"),
+        analytics.Attribute("method"),
+        analytics.Attribute("status_code", type=int),
+        analytics.Attribute("organization_id", type=int, required=False),
+        analytics.Attribute("organization_slug", required=False),
+        analytics.Attribute("project_id", type=int, required=False),
+        analytics.Attribute("project_slug", required=False),
+        analytics.Attribute("user_id", type=int),
+    )
+
+
 analytics.register(OrganizationSavedSearchCreatedEvent)
 analytics.register(OrganizationSavedSearchDeletedEvent)
 analytics.register(GroupSimilarIssuesEmbeddingsCountEvent)
+analytics.register(DevToolbarApiRequestEvent)

src/sentry/conf/server.py

@@ -346,6 +346,7 @@ def env(
     "sentry.middleware.locale.SentryLocaleMiddleware",
     "sentry.middleware.ratelimit.RatelimitMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
+    "sentry.middleware.devtoolbar.DevToolbarAnalyticsMiddleware",
 )
 
 ROOT_URLCONF = "sentry.conf.urls"

src/sentry/middleware/devtoolbar.py

@@ -0,0 +1,62 @@
+import logging
+
+from django.http import HttpRequest, HttpResponse
+
+from sentry import analytics, options
+from sentry.utils.http import origin_from_request
+from sentry.utils.http import query_string as get_query_string
+from sentry.utils.urls import parse_id_or_slug_param
+
+logger = logging.getLogger(__name__)
+
+
+class DevToolbarAnalyticsMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+        try:
+            # note ordering of conditions to avoid extra option queries.
+            if request.headers.get("queryReferrer") == "devtoolbar" and options.get(
+                "devtoolbar.analytics.enabled"
+            ):
+                _record_api_request(request, response)
+        except Exception:
+            logger.exception("devtoolbar: exception while recording api analytics event.")
+
+        return response
+
+
+def _record_api_request(request: HttpRequest, response: HttpResponse) -> None:
+    resolver_match = request.resolver_match
+    if resolver_match is None:
+        raise ValueError(f"Request URL not resolved: {request.path_info}")
+
+    kwargs, route, view_name = (
+        resolver_match.kwargs,
+        resolver_match.route,
+        resolver_match.view_name,
+    )
+
+    org_id_or_slug = kwargs.get("organization_id_or_slug", kwargs.get("organization_slug"))
+    org_id, org_slug = parse_id_or_slug_param(org_id_or_slug)
+    project_id_or_slug = kwargs.get("project_id_or_slug")
+    project_id, project_slug = parse_id_or_slug_param(project_id_or_slug)
+
+    origin = origin_from_request(request)
+    query_string: str = get_query_string(request)  # starts with '?'
+    analytics.record(
+        "devtoolbar.api_request",
+        view_name=view_name,
+        route=route,
+        query_string=query_string,
+        origin=origin,
+        method=request.method,
+        status_code=response.status_code,
+        organization_id=org_id or None,
+        organization_slug=org_slug,
+        project_id=project_id or None,
+        project_slug=project_slug,
+        user_id=request.user.id if hasattr(request, "user") and request.user else None,
+    )

src/sentry/options/defaults.py

@@ -487,6 +487,14 @@
     flags=FLAG_ALLOW_EMPTY | FLAG_AUTOMATOR_MODIFIABLE,
 )
 
+# Dev Toolbar Options
+register(
+    "devtoolbar.analytics.enabled",
+    type=Bool,
+    default=False,
+    flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE,
+)
+
 
 # Extract spans only from a random fraction of transactions.
 #

src/sentry/utils/urls.py

@@ -73,3 +73,12 @@ def urlsplit_best_effort(s: str) -> tuple[str, str, str, str]:
         return scheme, netloc, path, query
     else:
         return parsed.scheme, parsed.netloc, parsed.path, parsed.query
+
+
+def parse_id_or_slug_param(id_or_slug: str | None) -> tuple[int | None, str | None]:
+    if not id_or_slug:
+        return None, None
+
+    if id_or_slug.isnumeric():
+        return int(id_or_slug), None
+    return None, id_or_slug

tests/sentry/middleware/test_devtoolbar.py

@@ -0,0 +1,245 @@
+from functools import cached_property
+from unittest.mock import MagicMock, patch
+
+from django.http import HttpResponse
+from django.test import RequestFactory, override_settings
+
+from sentry.api import DevToolbarApiRequestEvent
+from sentry.middleware.devtoolbar import DevToolbarAnalyticsMiddleware
+from sentry.testutils.cases import APITestCase, SnubaTestCase, TestCase
+from sentry.testutils.helpers import override_options
+from sentry.types.group import GroupSubStatus
+
+
+class DevToolbarAnalyticsMiddlewareUnitTest(TestCase):
+    middleware = cached_property(DevToolbarAnalyticsMiddleware)
+    analytics_event_name = DevToolbarApiRequestEvent.type
+
+    @cached_property
+    def factory(self):
+        return RequestFactory()
+
+    def setUp(self):
+        # Allows changing the get_response mock for each test.
+        self.get_response = MagicMock(return_value=HttpResponse(status=200))
+        self.middleware.get_response = self.get_response
+
+    def make_toolbar_request(
+        self,
+        path="/",
+        method="GET",
+        headers=None,
+        incl_toolbar_header=True,
+        resolver_match="mock",
+    ):
+        headers = headers or {}
+        if incl_toolbar_header:
+            headers["queryReferrer"] = "devtoolbar"
+        request = getattr(self.factory, method.lower())(path, headers=headers)
+        request.resolver_match = MagicMock() if resolver_match == "mock" else resolver_match
+        return request
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_basic(self, mock_record: MagicMock):
+        request = self.make_toolbar_request()
+        self.middleware(request)
+        mock_record.assert_called()
+        assert mock_record.call_args[0][0] == self.analytics_event_name
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_no_devtoolbar_header(self, mock_record: MagicMock):
+        request = self.make_toolbar_request(incl_toolbar_header=False)
+        self.middleware(request)
+        mock_record.assert_not_called()
+
+        request = self.make_toolbar_request(
+            headers={"queryReferrer": "not-toolbar"}, incl_toolbar_header=False
+        )
+        self.middleware(request)
+        mock_record.assert_not_called()
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.middleware.devtoolbar.logger.exception")
+    @patch("sentry.analytics.record")
+    def test_request_not_resolved(self, mock_record: MagicMock, mock_logger: MagicMock):
+        request = self.make_toolbar_request()
+        request.resolver_match = None
+        self.middleware(request)
+
+        mock_record.assert_not_called()
+        mock_logger.assert_called()
+
+    #################
+    # Attribute tests
+    #################
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_view_name_and_route(self, mock_record: MagicMock):
+        # Integration tests do a better job of testing these fields, since they involve route resolver.
+        view_name = "my-endpoint"
+        route = "/issues/(?P<issue_id>)/"
+        request = self.make_toolbar_request(
+            resolver_match=MagicMock(view_name=view_name, route=route),
+        )
+        self.middleware(request)
+
+        mock_record.assert_called()
+        assert mock_record.call_args[0][0] == self.analytics_event_name
+        assert mock_record.call_args[1].get("view_name") == view_name
+        assert mock_record.call_args[1].get("route") == route
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_query_string(self, mock_record: MagicMock):
+        query = "?a=b&statsPeriod=14d"
+        request = self.make_toolbar_request(
+            path="https://sentry.io/replays/" + query,
+        )
+        self.middleware(request)
+
+        mock_record.assert_called()
+        assert mock_record.call_args[0][0] == self.analytics_event_name
+        assert mock_record.call_args[1].get("query_string") == query
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_origin(self, mock_record: MagicMock):
+        origin = "https://potato.com"
+        request = self.make_toolbar_request(headers={"Origin": origin})
+        self.middleware(request)
+
+        mock_record.assert_called()
+        assert mock_record.call_args[0][0] == self.analytics_event_name
+        assert mock_record.call_args[1].get("origin") == origin
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_origin_from_referrer(self, mock_record: MagicMock):
+        origin = "https://potato.com"
+        request = self.make_toolbar_request(headers={"Referer": origin + "/issues/?a=b"})
+        self.middleware(request)
+
+        mock_record.assert_called()
+        assert mock_record.call_args[0][0] == self.analytics_event_name
+        assert mock_record.call_args[1].get("origin") == origin
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_response_status_code(self, mock_record: MagicMock):
+        request = self.make_toolbar_request()
+        self.get_response.return_value = HttpResponse(status=420)
+        self.middleware(request)
+
+        mock_record.assert_called()
+        assert mock_record.call_args[0][0] == self.analytics_event_name
+        assert mock_record.call_args[1].get("status_code") == 420
+
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def test_methods(self, mock_record: MagicMock):
+        for method in ["GET", "POST", "PUT", "DELETE"]:
+            request = self.make_toolbar_request(method=method)
+            self.middleware(request)
+
+            mock_record.assert_called()
+            assert mock_record.call_args[0][0] == self.analytics_event_name
+            assert mock_record.call_args[1].get("method") == method
+
+
+TEST_MIDDLEWARE = (
+    "django.middleware.common.CommonMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "sentry.middleware.auth.AuthenticationMiddleware",
+    "sentry.middleware.devtoolbar.DevToolbarAnalyticsMiddleware",
+)
+
+
+class DevToolbarAnalyticsMiddlewareIntegrationTest(APITestCase, SnubaTestCase):
+    def setUp(self):
+        super().setUp()
+        self.login_as(user=self.user)
+        self.origin = "https://third-party.site.com"
+
+    @override_settings(MIDDLEWARE=TEST_MIDDLEWARE)
+    @override_options({"devtoolbar.analytics.enabled": True})
+    @patch("sentry.analytics.record")
+    def _test_endpoint(
+        self,
+        path: str,
+        query_string: str,
+        method: str,
+        expected_view_name: str,
+        expected_route: str,
+        mock_record: MagicMock,
+        expected_org_id=None,
+        expected_org_slug=None,
+        expected_proj_id=None,
+        expected_proj_slug=None,
+    ):
+        url = path + query_string
+        response: HttpResponse = getattr(self.client, method.lower())(
+            url,
+            headers={
+                "queryReferrer": "devtoolbar",
+                "Origin": self.origin,
+            },
+        )
+
+        mock_record.assert_any_call(
+            "devtoolbar.api_request",
+            view_name=expected_view_name,
+            route=expected_route,
+            query_string=query_string,
+            origin=self.origin,
+            method=method,
+            status_code=response.status_code,
+            organization_id=expected_org_id,
+            organization_slug=expected_org_slug,
+            project_id=expected_proj_id,
+            project_slug=expected_proj_slug,
+            user_id=self.user.id,
+        )
+
+    def test_organization_replays(self):
+        self._test_endpoint(
+            f"/api/0/organizations/{self.organization.slug}/replays/",
+            "?field=id",
+            "GET",
+            "sentry-api-0-organization-replay-index",
+            "^api/0/organizations/(?P<organization_id_or_slug>[^\\/]+)/replays/$",
+            expected_org_slug=self.organization.slug,
+        )
+        self._test_endpoint(
+            f"/api/0/organizations/{self.organization.id}/replays/",
+            "?field=id",
+            "GET",
+            "sentry-api-0-organization-replay-index",
+            "^api/0/organizations/(?P<organization_id_or_slug>[^\\/]+)/replays/$",
+            expected_org_id=self.organization.id,
+        )
+
+    def test_group_details(self):
+        group = self.create_group(substatus=GroupSubStatus.NEW)
+        self._test_endpoint(
+            f"/api/0/organizations/{self.organization.slug}/issues/{group.id}/",
+            "",
+            "GET",
+            "sentry-api-0-organization-group-group-details",
+            "^api/0/organizations/(?P<organization_id_or_slug>[^\\/]+)/(?:issues|groups)/(?P<issue_id>[^\\/]+)/$",
+            expected_org_slug=self.organization.slug,
+        )
+
+    def test_project_user_feedback(self):
+        # Should return 400 (no POST data)
+        self._test_endpoint(
+            f"/api/0/projects/{self.organization.slug}/{self.project.id}/user-feedback/",
+            "",
+            "POST",
+            "sentry-api-0-project-user-reports",
+            r"^api/0/projects/(?P<organization_id_or_slug>[^\/]+)/(?P<project_id_or_slug>[^\/]+)/(?:user-feedback|user-reports)/$",
+            expected_org_slug=self.organization.slug,
+            expected_proj_id=self.project.id,
+        )