chore: initialise HTTP client when cloning modules (#14777)

## Description

A recent attempt to reduce start-up time by lazy-loading the HTTP client
modules resulted in a potential gevent support regression whereby the
lazily created HTTP client could end up making requests via the
gevent-monkey-patched socket module. This introduces the possibility of
spurious deadlock during the processing of requests. We prevent this by
force-importing the HTTP clients on bootstrap to ensure that we grab
references to an original copy of the socket module. This prevents our
HTTP clients from calling into gevent when uploading payloads to the
agent, preventing the potential deadlocks.

Author: P403n1x87

ddtrace/appsec/_asm_request_context.py

@@ -2,6 +2,7 @@
 import json
 import re
 from types import TracebackType
+from typing import TYPE_CHECKING
 from typing import Any
 from typing import Callable
 from typing import Dict
@@ -17,8 +18,6 @@
 from ddtrace.appsec._constants import APPSEC
 from ddtrace.appsec._constants import SPAN_DATA_NAMES
 from ddtrace.appsec._constants import Constant_Class
-from ddtrace.appsec._utils import DDWaf_info
-from ddtrace.appsec._utils import DDWaf_result
 from ddtrace.appsec._utils import Telemetry_result
 from ddtrace.appsec._utils import get_triggers
 from ddtrace.contrib.internal.trace_utils_base import _normalize_tag_name
@@ -29,6 +28,10 @@
 from ddtrace.settings.asm import config as asm_config
 
 
+if TYPE_CHECKING:
+    from ddtrace.appsec._utils import DDWaf_info
+    from ddtrace.appsec._utils import DDWaf_result
+
 logger = ddlogger.get_logger(__name__)
 
 
@@ -95,7 +98,7 @@ def __init__(self, span: Optional[Span] = None, rc_products: str = ""):
         else:
             self.framework = self.span.name
         self.framework = self.framework.lower().replace(" ", "_")
-        self.waf_info: Optional[Callable[[], DDWaf_info]] = None
+        self.waf_info: Optional[Callable[[], "DDWaf_info"]] = None
         self.waf_addresses: Dict[str, Any] = {}
         self.callbacks: Dict[str, Any] = {_CONTEXT_CALL: []}
         self.telemetry: Telemetry_result = Telemetry_result()
@@ -375,15 +378,15 @@ def set_waf_callback(value) -> None:
     set_value(_CALLBACKS, _WAF_CALL, value)
 
 
-def set_waf_info(info: Callable[[], DDWaf_info]) -> None:
+def set_waf_info(info: Callable[[], "DDWaf_info"]) -> None:
     env = _get_asm_context()
     if env is None:
         logger.warning(WARNING_TAGS.SET_WAF_INFO_NO_ASM_CONTEXT, extra=log_extra, stack_info=True)
         return
     env.waf_info = info
 
 
-def call_waf_callback(custom_data: Optional[Dict[str, Any]] = None, **kwargs) -> Optional[DDWaf_result]:
+def call_waf_callback(custom_data: Optional[Dict[str, Any]] = None, **kwargs) -> Optional["DDWaf_result"]:
     if not asm_config._asm_enabled:
         return None
     callback = get_value(_CALLBACKS, _WAF_CALL)
@@ -480,7 +483,7 @@ def asm_request_context_set(
 def set_waf_telemetry_results(
     rules_version: str,
     is_blocked: bool,
-    waf_results: DDWaf_result,
+    waf_results: "DDWaf_result",
     rule_type: Optional[str],
     is_sampled: bool,
 ) -> None:

ddtrace/appsec/_utils.py

@@ -10,7 +10,6 @@
 from typing import List
 from typing import Optional
 from typing import Union
-import uuid
 
 from ddtrace.appsec._constants import API_SECURITY
 from ddtrace.appsec._constants import APPSEC
@@ -236,6 +235,10 @@ def _safe_userid(user_id):
         return user_id
     except ValueError:
         try:
+            # Import uuid lazily because this also imports threading via the
+            # platform module
+            import uuid
+
             _ = uuid.UUID(user_id)
             return user_id
         except ValueError:

ddtrace/bootstrap/sitecustomize.py

@@ -55,6 +55,11 @@ def drop(module_name):
     if not asbool(do_cleanup):
         return
 
+    # We need to import these modules to make sure they grab references to the
+    # right modules before we start unloading stuff.
+    import ddtrace.internal.http  # noqa
+    import ddtrace.internal.uds  # noqa
+
     # Unload all the modules that we have imported, except for the ddtrace one.
     # NB: this means that every `import threading` anywhere in `ddtrace/` code
     # uses a copy of that module that is distinct from the copy that user code
@@ -94,6 +99,10 @@ def drop(module_name):
             # submodule makes use of threading so it is critical to unload when
             # gevent is used.
             "concurrent.futures",
+            # We unload the threading module in case it was imported by
+            # CPython on boot.
+            "threading",
+            "_thread",
         ]
     )
     for u in UNLOAD_MODULES:

tests/internal/test_auto.py

@@ -1,18 +1,39 @@
-import sys
-
 import pytest
 
 
-@pytest.mark.skipif(sys.version_info < (3, 12, 5), reason="Python < 3.12.5 eagerly loads the threading module")
+# DEV: This test must pass ALWAYS. If this test fails, it means that something
+# needs to be fixed somewhere. Please DO NOT skip this test under any
+# circumstance!
 @pytest.mark.subprocess(
     env=dict(
         DD_UNLOAD_MODULES_FROM_SITECUSTOMIZE="true",
-        DD_REMOTE_CONFIGURATION_ENABLED="1",
-    )
+    ),
+    parametrize={
+        "DD_REMOTE_CONFIGURATION_ENABLED": ("1", "0"),
+    },
 )
 def test_auto():
+    import os
     import sys
 
     import ddtrace.auto  # noqa:F401
 
     assert "threading" not in sys.modules
+    assert "socket" not in sys.modules
+    assert "subprocess" not in sys.modules
+
+    if os.getenv("DD_REMOTE_CONFIGURATION_ENABLED") == "0":
+        # When unloading modules we must have the HTTP clients imported already
+        assert "ddtrace.internal.http" in sys.modules
+
+        # emulate socket patching (e.g. by gevent)
+        import socket  # noqa:F401
+
+        socket.create_connection = None
+        socket.socket = None
+
+        from ddtrace.internal.http import HTTPConnection  # noqa:F401
+        import ddtrace.internal.uds as uds
+
+        assert HTTPConnection("localhost")._create_connection is not None
+        assert uds.socket.socket is not None