Merge branch 'baptiste.foy/FA/core-to-native' into baptiste.foy/FA/li…

…brary-config-libdatadog

.github/workflows/build-and-publish-image.yml

@@ -26,6 +26,9 @@ on:
 jobs:
   build_push:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/build_python_3.yml

@@ -127,7 +127,7 @@ jobs:
           CIBW_BEFORE_ALL: >
             if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]];
             then
-              curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y;
+              curl -sSf https://sh.rustup.rs | sh -s -- -y;
             fi
           CIBW_BEFORE_ALL_WINDOWS: rustup target add i686-pc-windows-msvc
           CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin

.github/workflows/pypa_musllinux_1_2_i686.yml

@@ -12,6 +12,9 @@ on:
 jobs:
   build-and-publish:
     uses: ./.github/workflows/build-and-publish-image.yml
+    permissions:
+      contents: read
+      packages: write
     with:
       tags: 'ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:${{ github.sha }},ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest'
       platforms: 'linux/386'

.gitlab/benchmarks.yml

@@ -25,7 +25,6 @@ variables:
     paths:
       - reports/
     expire_in: 3 months
-  allow_failure: true  # Allow failure, so partial results are uploaded
   variables:
     UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
     UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-py"
@@ -78,6 +77,21 @@ benchmarks-pr-comment:
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
     KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-py
 
+check-big-regressions:
+  stage: benchmarks
+  needs: [ microbenchmarks, benchmark-serverless ]
+  when: always
+  tags: ["arch:amd64"]
+  image: $MICROBENCHMARKS_CI_IMAGE
+  script:
+    - export ARTIFACTS_DIR="$(pwd)/reports/"
+    - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
+    - git clone --branch dd-trace-py https://github.com/DataDog/benchmarking-platform /platform && cd /platform
+    - bp-runner bp-runner.fail-on-regression.yml --debug
+  variables:
+    # Gitlab and BP specific env vars. Do not modify.
+    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-py
+
 benchmark-serverless:
   stage: benchmarks
   image: $SLS_CI_IMAGE

ddtrace/_monkey.py

@@ -176,9 +176,13 @@ def _on_import_factory(module, prefix="ddtrace.contrib", raise_errors=True, patc
 
     def on_import(hook):
         # Import and patch module
-        path = "%s.%s" % (prefix, module)
         try:
-            imported_module = importlib.import_module(path)
+            try:
+                imported_module = importlib.import_module("%s.internal.%s.patch" % (prefix, module))
+            except ImportError:
+                # Some integrations do not have an internal patch module, so we use the public one
+                # FIXME: This is a temporary solution until we refactor the patching logic.
+                imported_module = importlib.import_module("%s.%s" % (prefix, module))
             imported_module.patch()
             if hasattr(imported_module, "patch_submodules"):
                 imported_module.patch_submodules(patch_indicator)
@@ -204,7 +208,8 @@ def on_import(hook):
                     telemetry.telemetry_writer.add_integration(
                         name, True, PATCH_MODULES.get(module) is True, "", version=v
                     )
-            else:
+            elif hasattr(imported_module, "get_version"):
+                # TODO: Ensure every integration defines either get_version or get_versions in their patch.py module
                 version = imported_module.get_version()
                 telemetry.telemetry_writer.add_integration(
                     module, True, PATCH_MODULES.get(module) is True, "", version=version

ddtrace/_trace/trace_handlers.py

@@ -22,7 +22,7 @@
 from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
 from ddtrace.contrib.internal.botocore.constants import BOTOCORE_STEPFUNCTIONS_INPUT_KEY
-from ddtrace.contrib.trace_utils import _set_url_tag
+from ddtrace.contrib.internal.trace_utils import _set_url_tag
 from ddtrace.ext import SpanKind
 from ddtrace.ext import db
 from ddtrace.ext import http

ddtrace/_trace/utils_redis.py

@@ -10,7 +10,7 @@
 from ddtrace.constants import SPAN_KIND
 from ddtrace.constants import SPAN_MEASURED_KEY
 from ddtrace.contrib import trace_utils
-from ddtrace.contrib.redis_utils import _extract_conn_tags
+from ddtrace.contrib.internal.redis_utils import _extract_conn_tags
 from ddtrace.ext import SpanKind
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import db

ddtrace/appsec/_deduplications.py

@@ -28,9 +28,12 @@ def _reset_cache(self):
         """
         self.reported_logs.clear()
 
+    def _check_deduplication(self):
+        return asm_config._asm_deduplication_enabled
+
     def __call__(self, *args, **kwargs):
         result = None
-        if asm_config._deduplication_enabled:
+        if self._check_deduplication():
             raw_log_hash = hash("".join([str(arg) for arg in self._extract(args)]))
             last_reported_timestamp = self.reported_logs.get(raw_log_hash, M_INF) + self._time_lapse
             current = monotonic()

ddtrace/appsec/_handlers.py

@@ -6,8 +6,8 @@
 from ddtrace.appsec._asm_request_context import get_blocked
 from ddtrace.appsec._constants import SPAN_DATA_NAMES
 from ddtrace.contrib import trace_utils
-from ddtrace.contrib.trace_utils import _get_request_header_user_agent
-from ddtrace.contrib.trace_utils import _set_url_tag
+from ddtrace.contrib.internal.trace_utils import _get_request_header_user_agent
+from ddtrace.contrib.internal.trace_utils import _set_url_tag
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import http
 from ddtrace.internal import core

ddtrace/appsec/_iast/_handlers.py

@@ -297,7 +297,6 @@ def if_iast_taint_yield_tuple_for(origins, wrapped, instance, args, kwargs):
 
 def if_iast_taint_returned_object_for(origin, wrapped, instance, args, kwargs):
     value = wrapped(*args, **kwargs)
-
     if _is_iast_enabled() and is_iast_request_enabled():
         try:
             if not is_pyobject_tainted(value):
@@ -310,6 +309,29 @@ def if_iast_taint_returned_object_for(origin, wrapped, instance, args, kwargs):
     return value
 
 
+def if_iast_taint_starlette_datastructures(origin, wrapped, instance, args, kwargs):
+    value = wrapped(*args, **kwargs)
+    if _is_iast_enabled() and is_iast_request_enabled():
+        try:
+            res = []
+            for element in value:
+                if not is_pyobject_tainted(element):
+                    res.append(
+                        taint_pyobject(
+                            pyobject=element,
+                            source_name=origin_to_str(origin),
+                            source_value=element,
+                            source_origin=origin,
+                        )
+                    )
+                else:
+                    res.append(element)
+            return res
+        except Exception:
+            log.debug("Unexpected exception while tainting pyobject", exc_info=True)
+    return value
+
+
 def _on_iast_fastapi_patch():
     # Cookies sources
     try_wrap_function_wrapper(
@@ -333,6 +355,13 @@ def _on_iast_fastapi_patch():
     )
     _set_metric_iast_instrumented_source(OriginType.PARAMETER)
 
+    try_wrap_function_wrapper(
+        "starlette.datastructures",
+        "QueryParams.keys",
+        functools.partial(if_iast_taint_starlette_datastructures, OriginType.PARAMETER_NAME),
+    )
+    _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
+
     # Header sources
     try_wrap_function_wrapper(
         "starlette.datastructures",
@@ -346,6 +375,13 @@ def _on_iast_fastapi_patch():
     )
     _set_metric_iast_instrumented_source(OriginType.HEADER)
 
+    try_wrap_function_wrapper(
+        "starlette.datastructures",
+        "Headers.keys",
+        functools.partial(if_iast_taint_starlette_datastructures, OriginType.HEADER_NAME),
+    )
+    _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
+
     # Path source
     try_wrap_function_wrapper("starlette.datastructures", "URL.__init__", _iast_instrument_starlette_url)
     _set_metric_iast_instrumented_source(OriginType.PATH)
@@ -363,6 +399,12 @@ def _on_iast_fastapi_patch():
         "FormData.get",
         functools.partial(if_iast_taint_returned_object_for, OriginType.BODY),
     )
+    try_wrap_function_wrapper(
+        "starlette.datastructures",
+        "FormData.keys",
+        functools.partial(if_iast_taint_starlette_datastructures, OriginType.PARAMETER_NAME),
+    )
+
     _set_metric_iast_instrumented_source(OriginType.BODY)
 
     # Instrumented on _iast_starlette_scope_taint

ddtrace/appsec/_iast/taint_sinks/_base.py

@@ -5,10 +5,11 @@
 from typing import Text
 
 from ddtrace import tracer
+from ddtrace.appsec._deduplications import deduplication
 from ddtrace.appsec._trace_utils import _asm_manual_keep
 from ddtrace.internal.logger import get_logger
+from ddtrace.settings.asm import config as asm_config
 
-from ..._deduplications import deduplication
 from .._iast_request_context import get_iast_reporter
 from .._iast_request_context import is_iast_request_enabled
 from .._iast_request_context import set_iast_reporter
@@ -27,6 +28,9 @@
 
 
 class taint_sink_deduplication(deduplication):
+    def _check_deduplication(self):
+        return asm_config._iast_deduplication_enabled
+
     def _extract(self, args):
         # We skip positions 0 and 1 because they represent the 'cls' and 'span' respectively
         return args[2:]

ddtrace/appsec/_processor.py

@@ -101,7 +101,7 @@ def get_rules() -> str:
 
 
 def _set_headers(span: Span, headers: Any, kind: str, only_asm_enabled: bool = False) -> None:
-    from ddtrace.contrib.trace_utils import _normalize_tag_name
+    from ddtrace.contrib.internal.trace_utils import _normalize_tag_name
 
     for k in headers:
         if isinstance(k, tuple):
@@ -218,7 +218,7 @@ def rasp_sqli_enabled(self) -> bool:
         return WAF_DATA_NAMES.SQLI_ADDRESS in self._addresses_to_keep
 
     def on_span_start(self, span: Span) -> None:
-        from ddtrace.contrib import trace_utils
+        from ddtrace.contrib.internal import trace_utils
 
         if not hasattr(self, "_ddwaf"):
             self.delayed_init()

ddtrace/appsec/_trace_utils.py

@@ -11,7 +11,7 @@
 from ddtrace.appsec._constants import LOGIN_EVENTS_MODE
 from ddtrace.appsec._constants import WAF_ACTIONS
 from ddtrace.appsec._utils import _hash_user_id
-from ddtrace.contrib.trace_utils import set_user
+from ddtrace.contrib.internal.trace_utils import set_user
 from ddtrace.ext import SpanTypes
 from ddtrace.ext import user
 from ddtrace.internal import core

ddtrace/appsec/_utils.py

@@ -22,7 +22,7 @@ def parse_response_body(raw_body):
 
     from ddtrace.appsec import _asm_request_context
     from ddtrace.appsec._constants import SPAN_DATA_NAMES
-    from ddtrace.contrib.trace_utils import _get_header_value_case_insensitive
+    from ddtrace.contrib.internal.trace_utils import _get_header_value_case_insensitive
 
     if not raw_body:
         return

ddtrace/contrib/__init__.py

@@ -1,4 +1,73 @@
 from ddtrace._trace import trace_handlers  # noqa:F401
+from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
 from ddtrace.internal.utils.importlib import func_name  # noqa:F401
 from ddtrace.internal.utils.importlib import module_name  # noqa:F401
 from ddtrace.internal.utils.importlib import require_modules  # noqa:F401
+from ddtrace.vendor.debtcollector import deprecate
+
+
+def __getattr__(name):
+    if name in (
+        "aiohttp",
+        "asgi",
+        "bottle",
+        "celery",
+        "cherrypy",
+        "falcon",
+        "flask_cache",
+        "pylibmc",
+        "pyramid",
+        "requests",
+        "sqlalchemy",
+        "wsgi",
+        "trace_utils",
+        "internal",
+    ):
+        # following packages/modules are not deprecated and will not be removed in 3.0
+        pass
+    elif name in ("trace_handlers", "func_name", "module_name", "require_modules"):
+        # the following attributes are exposed in ddtrace.contrib.__init__ and should be
+        # removed in v3.0
+        deprecate(
+            ("ddtrace.contrib.%s is deprecated" % name),
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+    elif name in ("aiobotocore", "httplib", "kombu", "snowflake", "sqlalchemy", "tornado", "urllib3"):
+        # following integrations are not enabled by default and require a unique deprecation message
+        deprecate(
+            f"ddtrace.contrib.{name} is deprecated",
+            message="Avoid using this package directly. "
+            f"Set DD_TRACE_{name.upper()}_ENABLED=true and use ``ddtrace.auto`` or the "
+            "``ddtrace-run`` command to enable and configure this integration.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+    elif name in ("redis_utils", "trace_utils_redis", "trace_utils_async"):
+        deprecate(
+            f"The ddtrace.contrib.{name} module is deprecated",
+            message="Import from ``ddtrace.contrib.trace_utils`` instead.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+    elif name == "flask_login":
+        deprecate(
+            """The flask_login integration is deprecated and will be deleted.
+        We recommend customers to switch to manual instrumentation.
+        https://docs.datadoghq.com/security/application_security/threats/add-user-info/?tab=loginsuccess&code-lang=python#adding-business-logic-information-login-success-login-failure-any-business-logic-to-traces
+        """,
+            message="",
+            category=DDTraceDeprecationWarning,
+        )
+    else:
+        deprecate(
+            f"ddtrace.contrib.{name} is deprecated",
+            message="Avoid using this package directly. "
+            f"Use ``import ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure {name}.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
+
+    if name in globals():
+        return globals()[name]
+    raise AttributeError("%s has no attribute %s", __name__, name)

ddtrace/contrib/aiobotocore/__init__.py

@@ -35,9 +35,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-# Expose public methods
-from ddtrace.contrib.internal.aiobotocore.patch import get_version
-from ddtrace.contrib.internal.aiobotocore.patch import patch
 
-
-__all__ = ["patch", "get_version"]
+from ddtrace.contrib.internal.aiobotocore.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiobotocore.patch import patch  # noqa: F401

ddtrace/contrib/aiohttp/__init__.py

@@ -94,11 +94,25 @@ async def home_handler(request):
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 from ddtrace.contrib.internal.aiohttp.middlewares import trace_app
-from ddtrace.contrib.internal.aiohttp.patch import get_version
+from ddtrace.contrib.internal.aiohttp.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiohttp.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.aiohttp.patch import unpatch  # noqa: F401
+from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning
+from ddtrace.vendor.debtcollector import deprecate
 
-# Expose public methods
-from ddtrace.contrib.internal.aiohttp.patch import patch
-from ddtrace.contrib.internal.aiohttp.patch import unpatch
 
+def __getattr__(name):
+    if name in ("patch", "get_version", "unpatch"):
+        deprecate(
+            ("%s.%s is deprecated" % (__name__, name)),
+            message="Use ``import ddtrace.auto`` or the ``ddtrace-run`` command to configure this integration.",
+            category=DDTraceDeprecationWarning,
+            removal_version="3.0.0",
+        )
 
-__all__ = ["patch", "unpatch", "trace_app", "get_version"]
+    if name in globals():
+        return globals()[name]
+    raise AttributeError("%s has no attribute %s", __name__, name)
+
+
+__all__ = ["trace_app"]

ddtrace/contrib/aiohttp_jinja2/__init__.py

@@ -23,9 +23,6 @@
     _w.simplefilter("ignore", DeprecationWarning)
     from . import patch as _  # noqa: F401, I001
 
-from ddtrace.contrib.internal.aiohttp_jinja2.patch import get_version
-from ddtrace.contrib.internal.aiohttp_jinja2.patch import patch
-from ddtrace.contrib.internal.aiohttp_jinja2.patch import unpatch
-
-
-__all__ = ["patch", "unpatch", "get_version"]
+from ddtrace.contrib.internal.aiohttp_jinja2.patch import get_version  # noqa: F401
+from ddtrace.contrib.internal.aiohttp_jinja2.patch import patch  # noqa: F401
+from ddtrace.contrib.internal.aiohttp_jinja2.patch import unpatch  # noqa: F401