ref(derived_code_mappings): Rename to Automatic Source Code Configura…

…tion (#83313)

The original name of the feature was Derived Code Mappings, however, we
have plans to expand it to create other changes beyond code mappings

The integrations team is working to bring feature parity across
integrations, thus, this is a good time to decouple the code.

This first change focuses on renaming and moving files around, while the
following pull request focuses on refactoring and decoupling the code.

.github/CODEOWNERS

@@ -519,7 +519,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 
 ## Issues
 **/issues/**                                                @getsentry/issues
-*code_mappings*.py                                          @getsentry/issues
 /src/sentry/api/helpers/actionable_items_helper.py          @getsentry/issues
 /src/sentry/api/helpers/events.py                           @getsentry/issues
 /src/sentry/api/helpers/group_index/                        @getsentry/issues
@@ -539,6 +538,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/tasks/auto_ongoing_issues.py                    @getsentry/issues
 /src/sentry/tasks/auto_remove_inbox.py                      @getsentry/issues
 /src/sentry/tasks/auto_resolve_issues.py                    @getsentry/issues
+/src/sentry/tasks/auto_source_code_config.py                @getsentry/issues
 /src/sentry/tasks/check_new_issue_threshold_met.py          @getsentry/issues
 /src/sentry/tasks/clear_expired_resolutions.py              @getsentry/issues
 /src/sentry/tasks/clear_expired_rulesnoozes.py              @getsentry/issues
@@ -547,7 +547,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/tasks/commit_context.py                         @getsentry/issues
 /src/sentry/tasks/commits.py                                @getsentry/issues
 /src/sentry/tasks/delete_seer_grouping_records.py           @getsentry/issues
-/src/sentry/tasks/derive_code_mappings.py                   @getsentry/issues
 /src/sentry/tasks/embeddings_grouping/                      @getsentry/issues
 /src/sentry/tasks/groupowner.py                             @getsentry/issues
 /src/sentry/tasks/merge.py                                  @getsentry/issues
@@ -574,6 +573,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /tests/sentry/tasks/test_auto_ongoing_issues.py             @getsentry/issues
 /tests/sentry/tasks/test_auto_remove_inbox.py               @getsentry/issues
 /tests/sentry/tasks/test_auto_resolve_issues.py             @getsentry/issues
+/tests/sentry/tasks/test_auto_source_code_config.py         @getsentry/issues
 /tests/sentry/tasks/test_backfill_seer_grouping_records.py  @getsentry/issues
 /tests/sentry/tasks/test_check_new_issue_threshold_met.py   @getsentry/issues
 /tests/sentry/tasks/test_clear_expired_resolutions.py       @getsentry/issues
@@ -583,7 +583,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /tests/sentry/tasks/test_commit_context.py                  @getsentry/issues
 /tests/sentry/tasks/test_commits.py                         @getsentry/issues
 /tests/sentry/tasks/test_delete_seer_grouping_records.py    @getsentry/issues
-/tests/sentry/tasks/test_derive_code_mappings.py            @getsentry/issues
 /tests/sentry/tasks/test_groupowner.py                      @getsentry/issues
 /tests/sentry/tasks/test_merge.py                           @getsentry/issues
 /tests/sentry/tasks/test_post_process.py                    @getsentry/issues

pyproject.toml

@@ -400,6 +400,7 @@ module = [
     "sentry.issues",
     "sentry.issues.analytics",
     "sentry.issues.apps",
+    "sentry.issues.auto_source_code_config.*",
     "sentry.issues.constants",
     "sentry.issues.endpoints",
     "sentry.issues.endpoints.actionable_items",
@@ -411,6 +412,7 @@ module = [
     "sentry.issues.endpoints.group_similar_issues_embeddings",
     "sentry.issues.endpoints.group_tombstone",
     "sentry.issues.endpoints.group_tombstone_details",
+    "sentry.issues.endpoints.organization_derive_code_mappings",
     "sentry.issues.endpoints.organization_eventid",
     "sentry.issues.endpoints.organization_group_index",
     "sentry.issues.endpoints.organization_group_index_stats",

src/sentry/api/endpoints/group_ai_autofix.py

@@ -18,7 +18,7 @@
 from sentry.api.serializers import EventSerializer, serialize
 from sentry.autofix.utils import get_autofix_repos_from_project_code_mappings, get_autofix_state
 from sentry.eventstore.models import Event, GroupEvent
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.group import Group
 from sentry.models.project import Project
 from sentry.profiles.utils import get_from_profiling_service

src/sentry/api/endpoints/group_autofix_setup_check.py

@@ -14,7 +14,7 @@
 from sentry.autofix.utils import get_autofix_repos_from_project_code_mappings
 from sentry.constants import ObjectStatus
 from sentry.integrations.services.integration import integration_service
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.group import Group
 from sentry.models.organization import Organization
 from sentry.models.project import Project

src/sentry/api/endpoints/project_repo_path_parsing.py

@@ -12,7 +12,7 @@
 from sentry.integrations.base import IntegrationFeatures
 from sentry.integrations.manager import default_manager as integrations
 from sentry.integrations.services.integration import RpcIntegration, integration_service
-from sentry.integrations.utils.code_mapping import find_roots
+from sentry.issues.auto_source_code_config.code_mapping import find_roots
 from sentry.models.repository import Repository
 
 

src/sentry/api/endpoints/project_stacktrace_coverage.py

@@ -9,9 +9,9 @@
 from sentry.api.base import region_silo_endpoint
 from sentry.api.bases.project import ProjectEndpoint
 from sentry.api.serializers import serialize
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
 from sentry.integrations.utils.codecov import codecov_enabled, fetch_codecov_data
 from sentry.integrations.utils.stacktrace_link import get_stacktrace_config
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.issues.endpoints.project_stacktrace_link import generate_context
 from sentry.models.project import Project
 from sentry.utils import metrics

src/sentry/api/urls.py

@@ -177,6 +177,7 @@
     GroupSimilarIssuesEndpoint,
     GroupTombstoneDetailsEndpoint,
     GroupTombstoneEndpoint,
+    OrganizationDeriveCodeMappingsEndpoint,
     OrganizationGroupIndexEndpoint,
     OrganizationGroupIndexStatsEndpoint,
     OrganizationGroupSearchViewsEndpoint,
@@ -448,7 +449,6 @@
     OrganizationDashboardWidgetDetailsEndpoint,
 )
 from .endpoints.organization_dashboards import OrganizationDashboardsEndpoint
-from .endpoints.organization_derive_code_mappings import OrganizationDeriveCodeMappingsEndpoint
 from .endpoints.organization_details import OrganizationDetailsEndpoint
 from .endpoints.organization_environments import OrganizationEnvironmentsEndpoint
 from .endpoints.organization_event_details import OrganizationEventDetailsEndpoint

src/sentry/autofix/utils.py

@@ -7,7 +7,7 @@
 from django.conf import settings
 from pydantic import BaseModel
 
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.project import Project
 from sentry.models.repository import Repository
 from sentry.seer.signed_seer_api import get_seer_salted_url, sign_with_seer_secret

src/sentry/conf/server.py

@@ -819,7 +819,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.dynamic_sampling.tasks.sliding_window_org",
     "sentry.dynamic_sampling.tasks.utils",
     "sentry.dynamic_sampling.tasks.custom_rule_notifications",
-    "sentry.tasks.derive_code_mappings",
+    "sentry.tasks.auto_source_code_config",
     "sentry.ingest.transaction_clusterer.tasks",
     "sentry.tasks.auto_enable_codecov",
     "sentry.tasks.weekly_escalating_forecast",
@@ -982,7 +982,9 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     Queue("replays.delete_replay", routing_key="replays.delete_replay"),
     Queue("counters-0", routing_key="counters-0"),
     Queue("triggers-0", routing_key="triggers-0"),
+    # XXX: Temporarilty keep in place until we have migrated to the new queue
     Queue("derive_code_mappings", routing_key="derive_code_mappings"),
+    Queue("auto_source_code_config", routing_key="auto_source_code_config"),
     Queue("transactions.name_clusterer", routing_key="transactions.name_clusterer"),
     Queue("auto_enable_codecov", routing_key="auto_enable_codecov"),
     Queue("weekly_escalating_forecast", routing_key="weekly_escalating_forecast"),

src/sentry/features/temporary.py

@@ -32,6 +32,10 @@ def register_temporary_features(manager: FeatureManager):
     # Features that don't use resource scoping #
     ############################################
 
+    # FLAGPOLE NOTE:
+    # You won't be able to control system feature flags with flagpole, as flagpole only handles
+    # organization or project scoped features. You would need to use an option instead.
+
     # Enables user registration.
     manager.add("auth:register", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True)
     # Enable creating organizations within sentry (if SENTRY_SINGLE_ORGANIZATION is not enabled).

src/sentry/integrations/github/client.py

@@ -26,7 +26,7 @@
 )
 from sentry.integrations.source_code_management.repository import RepositoryClient
 from sentry.integrations.types import EXTERNAL_PROVIDERS, ExternalProviders
-from sentry.integrations.utils.code_mapping import (
+from sentry.issues.auto_source_code_config.code_mapping import (
     MAX_CONNECTION_ERRORS,
     Repo,
     RepoTree,

src/sentry/integrations/github/integration.py

@@ -32,11 +32,11 @@
 from sentry.integrations.source_code_management.commit_context import CommitContextIntegration
 from sentry.integrations.source_code_management.repository import RepositoryIntegration
 from sentry.integrations.tasks.migrate_repo import migrate_repo
-from sentry.integrations.utils.code_mapping import RepoTree
 from sentry.integrations.utils.metrics import (
     IntegrationPipelineViewEvent,
     IntegrationPipelineViewType,
 )
+from sentry.issues.auto_source_code_config.code_mapping import RepoTree
 from sentry.models.repository import Repository
 from sentry.organizations.absolute_url import generate_organization_url
 from sentry.organizations.services.organization import RpcOrganizationSummary, organization_service

src/sentry/integrations/utils/commit_context.py

@@ -18,7 +18,9 @@
     FileBlameInfo,
     SourceLineInfo,
 )
-from sentry.integrations.utils.code_mapping import convert_stacktrace_frame_path_to_source_path
+from sentry.issues.auto_source_code_config.code_mapping import (
+    convert_stacktrace_frame_path_to_source_path,
+)
 from sentry.models.commit import Commit
 from sentry.models.commitauthor import CommitAuthor
 from sentry.shared_integrations.exceptions import ApiError

src/sentry/integrations/utils/stacktrace_link.py

@@ -7,7 +7,9 @@
 from sentry.integrations.models.repository_project_path_config import RepositoryProjectPathConfig
 from sentry.integrations.services.integration import integration_service
 from sentry.integrations.source_code_management.repository import RepositoryIntegration
-from sentry.integrations.utils.code_mapping import convert_stacktrace_frame_path_to_source_path
+from sentry.issues.auto_source_code_config.code_mapping import (
+    convert_stacktrace_frame_path_to_source_path,
+)
 from sentry.models.repository import Repository
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.utils.event_frames import EventFrame

src/sentry/integrations/utils/code_mapping.py → src/sentry/issues/auto_source_code_config/code_mapping.py

@@ -13,6 +13,8 @@
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+SUPPORTED_LANGUAGES = ["javascript", "python", "node", "ruby", "php", "go", "csharp"]
+
 SLASH = "/"
 BACKSLASH = "\\"  # This is the Python representation of a single backslash
 
@@ -109,7 +111,9 @@ def __init__(self, frame_file_path: str) -> None:
     def __repr__(self) -> str:
         return f"FrameFilename: {self.full_path}"
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, FrameFilename):
+            return False
         return self.full_path == other.full_path
 
 

src/sentry/issues/endpoints/__init__.py

@@ -10,6 +10,7 @@
 from .group_similar_issues_embeddings import GroupSimilarIssuesEmbeddingsEndpoint
 from .group_tombstone import GroupTombstoneEndpoint
 from .group_tombstone_details import GroupTombstoneDetailsEndpoint
+from .organization_derive_code_mappings import OrganizationDeriveCodeMappingsEndpoint
 from .organization_eventid import EventIdLookupEndpoint
 from .organization_group_index import OrganizationGroupIndexEndpoint
 from .organization_group_index_stats import OrganizationGroupIndexStatsEndpoint
@@ -43,6 +44,7 @@
     "GroupSimilarIssuesEndpoint",
     "GroupTombstoneDetailsEndpoint",
     "GroupTombstoneEndpoint",
+    "OrganizationDeriveCodeMappingsEndpoint",
     "OrganizationGroupIndexEndpoint",
     "OrganizationGroupIndexStatsEndpoint",
     "OrganizationGroupSearchViewsEndpoint",

src/sentry/api/endpoints/organization_derive_code_mappings.py → src/sentry/issues/endpoints/organization_derive_code_mappings.py

@@ -11,7 +11,7 @@
 )
 from sentry.api.serializers import serialize
 from sentry.integrations.github.integration import GitHubIntegration
-from sentry.integrations.utils.code_mapping import (
+from sentry.issues.auto_source_code_config.code_mapping import (
     CodeMapping,
     CodeMappingTreesHelper,
     FrameFilename,
@@ -20,11 +20,16 @@
 )
 from sentry.models.organization import Organization
 from sentry.models.project import Project
-from sentry.tasks.derive_code_mappings import get_installation
+from sentry.tasks.auto_source_code_config import get_installation
 
 
 @region_silo_endpoint
 class OrganizationDeriveCodeMappingsEndpoint(OrganizationEndpoint):
+    """
+    In the UI, we have a prompt to derive code mappings from the stacktrace filename.
+    This endpoint is used to get the possible code mappings for it.
+    """
+
     owner = ApiOwner.ISSUES
     publish_status = {
         "GET": ApiPublishStatus.UNKNOWN,
@@ -42,7 +47,8 @@ def get(self, request: Request, organization: Organization) -> Response:
         :auth: required
         """
         stacktrace_filename = request.GET.get("stacktraceFilename")
-        installation, _ = get_installation(organization)  # only returns GitHub integrations
+        # It only returns the first GitHub integration
+        installation, _ = get_installation(organization)
         if not installation:
             return self.respond(
                 {"text": "Could not find this integration installed on your organization"},

src/sentry/issues/endpoints/project_stacktrace_link.py

@@ -17,8 +17,8 @@
 from sentry.integrations.api.serializers.models.integration import IntegrationSerializer
 from sentry.integrations.base import IntegrationFeatures
 from sentry.integrations.services.integration import integration_service
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
 from sentry.integrations.utils.stacktrace_link import StacktraceLinkOutcome, get_stacktrace_config
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.project import Project
 
 logger = logging.getLogger(__name__)

src/sentry/options/defaults.py

@@ -57,6 +57,7 @@
     default=2**31,
     flags=FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE,
 )
+register("system.new-auto-source-code-config-queue", default=False, flags=FLAG_AUTOMATOR_MODIFIABLE)
 
 # URL configuration
 # Absolute URL to the sentry root directory. Should not include a trailing slash.

src/sentry/tasks/derive_code_mappings.py → src/sentry/tasks/auto_source_code_config.py

@@ -17,7 +17,7 @@
     SCMIntegrationInteractionEvent,
     SCMIntegrationInteractionType,
 )
-from sentry.integrations.utils.code_mapping import CodeMapping, CodeMappingTreesHelper
+from sentry.issues.auto_source_code_config.code_mapping import CodeMapping, CodeMappingTreesHelper
 from sentry.locks import locks
 from sentry.models.organization import Organization
 from sentry.models.project import Project
@@ -27,8 +27,6 @@
 from sentry.utils.locking import UnableToAcquireLock
 from sentry.utils.safe import get_path
 
-SUPPORTED_LANGUAGES = ["javascript", "python", "node", "ruby", "php", "go", "csharp"]
-
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
@@ -81,6 +79,7 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None:
     )
 
 
+# XXX: To be deleted after queue is empty
 @instrumented_task(
     name="sentry.tasks.derive_code_mappings.derive_code_mappings",
     queue="derive_code_mappings",
@@ -91,11 +90,28 @@ def derive_code_mappings(
     project_id: int,
     data: NodeData | None = None,  # We will deprecate this
     event_id: str | None = None,
+    **kwargs: Any,
+) -> None:
+    auto_source_code_config(project_id, data=data, event_id=event_id, **kwargs)
+
+
+@instrumented_task(
+    name="sentry.tasks.auto_source_code_config",
+    queue="auto_source_code_config",
+    default_retry_delay=60 * 10,
+    max_retries=3,
+)
+def auto_source_code_config(
+    project_id: int,
+    data: NodeData,
+    event_id: str | None = None,
+    **kwargs: Any,
 ) -> None:
     """
-    Derive code mappings for a project given data from a recent event.
+    Process errors for customers with source code management installed and calculate code mappings
+    among other things.
 
-    This task is queued at most once per hour per project, based on the ingested events.
+    This task is queued at most once per hour per project.
     """
     project = Project.objects.get(id=project_id)
     org: Organization = Organization.objects.get(id=project.organization_id)
@@ -113,10 +129,6 @@ def derive_code_mappings(
             return
         data = event.data
 
-    if data is None:
-        logger.error("Data is None.", extra=extra)
-        return
-
     stacktrace_paths: list[str] = identify_stacktrace_paths(data)
     if not stacktrace_paths:
         logger.info("No stacktrace paths found.", extra=extra)

src/sentry/tasks/commit_context.py

@@ -13,11 +13,11 @@
 from sentry import analytics
 from sentry.api.serializers.models.release import get_users_for_authors
 from sentry.integrations.source_code_management.commit_context import CommitContextIntegration
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
 from sentry.integrations.utils.commit_context import (
     find_commit_context_for_event_all_frames,
     get_or_create_commit_from_blame,
 )
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.locks import locks
 from sentry.models.commit import Commit
 from sentry.models.commitauthor import CommitAuthor

src/sentry/tasks/post_process.py

@@ -994,7 +994,8 @@ def process_code_mappings(job: PostProcessJob) -> None:
     if job["is_reprocessed"]:
         return
 
-    from sentry.tasks.derive_code_mappings import SUPPORTED_LANGUAGES, derive_code_mappings
+    from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES
+    from sentry.tasks.auto_source_code_config import auto_source_code_config, derive_code_mappings
 
     try:
         event = job["event"]
@@ -1016,8 +1017,10 @@ def process_code_mappings(job: PostProcessJob) -> None:
         else:
             return
 
-        # XXX: We will stop calling data after we deploy this change
-        derive_code_mappings.delay(project.id, data=event.data, event_id=event.event_id)
+        if options.get("system.new-auto-source-code-config-queue"):
+            auto_source_code_config.delay(project.id, event.data)
+        else:
+            derive_code_mappings.delay(project.id, event.data)
 
     except Exception:
         logger.exception("derive_code_mappings: Failed to process code mappings")