Merge branch 'main' into evan.li/release-ragas

.github/CODEOWNERS

@@ -115,6 +115,8 @@ ddtrace/contrib/flask_login/        @DataDog/asm-python
 ddtrace/contrib/webbrowser          @DataDog/asm-python
 ddtrace/contrib/urllib              @DataDog/asm-python
 ddtrace/internal/_exceptions.py     @DataDog/asm-python
+ddtrace/internal/appsec/            @DataDog/asm-python
+ddtrace/internal/iast/              @DataDog/asm-python
 tests/appsec/                       @DataDog/asm-python
 tests/contrib/dbapi/test_dbapi_appsec.py    @DataDog/asm-python
 tests/contrib/subprocess            @DataDog/asm-python

.github/workflows/rust-ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        extension: ["src/core"]
+        extension: ["src/native"]
     steps:
       - uses: actions/checkout@v4
         with:

.gitlab-ci.yml

@@ -59,18 +59,20 @@ onboarding_tests_installer:
     matrix:
       - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine]
 
-
 onboarding_tests_k8s_injection:
   parallel:
     matrix:
-      - WEBLOG_VARIANT:
-          - dd-lib-python-init-test-django
-          - dd-lib-python-init-test-django-gunicorn
-          - dd-lib-python-init-test-django-gunicorn-alpine
-          - dd-lib-python-init-test-django-preinstalled
-          - dd-lib-python-init-test-django-unsupported-package-force
-          - dd-lib-python-init-test-django-uvicorn
-          - dd-lib-python-init-test-protobuf-old
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django, ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS, K8S_LIB_INJECTION_PROFILING_DISABLED, K8S_LIB_INJECTION_PROFILING_ENABLED, K8S_LIB_INJECTION_PROFILING_OVERRIDE]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-gunicorn, dd-lib-python-init-test-django-gunicorn-alpine, dd-lib-python-init-test-django-unsupported-package-force, dd-lib-python-init-test-django-uvicorn, dd-lib-python-init-test-protobuf-old ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_PROFILING_ENABLED]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-preinstalled]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
 
 deploy_to_di_backend:manual:
   stage: shared-pipeline

.gitlab/benchmarks.yml

@@ -25,7 +25,6 @@ variables:
     paths:
       - reports/
     expire_in: 3 months
-  allow_failure: true  # Allow failure, so partial results are uploaded
   variables:
     UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
     UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # "dd-trace-py"
@@ -78,6 +77,21 @@ benchmarks-pr-comment:
     UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
     KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-py
 
+check-big-regressions:
+  stage: benchmarks
+  needs: [ microbenchmarks, benchmark-serverless ]
+  when: always
+  tags: ["arch:amd64"]
+  image: $MICROBENCHMARKS_CI_IMAGE
+  script:
+    - export ARTIFACTS_DIR="$(pwd)/reports/"
+    - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
+    - git clone --branch dd-trace-py https://github.com/DataDog/benchmarking-platform /platform && cd /platform
+    - bp-runner bp-runner.fail-on-regression.yml --debug
+  variables:
+    # Gitlab and BP specific env vars. Do not modify.
+    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-py
+
 benchmark-serverless:
   stage: benchmarks
   image: $SLS_CI_IMAGE

benchmarks/http_propagation_extract/scenario.py

@@ -29,6 +29,8 @@ def generate_headers(self):
     def run(self):
         if self.styles:
             config._propagation_style_extract = self.styles.split(",") if ("," in self.styles) else [self.styles]
+            if "none" in config._propagation_style_extract:
+                config._propagation_style_extract.remove("none")
 
         headers = self.generate_headers()
 

ddtrace/appsec/_deduplications.py

@@ -28,9 +28,12 @@ def _reset_cache(self):
         """
         self.reported_logs.clear()
 
+    def _check_deduplication(self):
+        return asm_config._asm_deduplication_enabled
+
     def __call__(self, *args, **kwargs):
         result = None
-        if asm_config._deduplication_enabled:
+        if self._check_deduplication():
             raw_log_hash = hash("".join([str(arg) for arg in self._extract(args)]))
             last_reported_timestamp = self.reported_logs.get(raw_log_hash, M_INF) + self._time_lapse
             current = monotonic()

ddtrace/appsec/_iast/_ast/ast_patching.py

@@ -33,6 +33,7 @@
     "beautifulsoup4.",
     "cachetools.",
     "cryptography.",
+    "django.",
     "docutils.",
     "idna.",
     "iniconfig.",

ddtrace/appsec/_iast/_handlers.py

@@ -5,7 +5,8 @@
 from wrapt import wrap_function_wrapper as _w
 
 from ddtrace.appsec._iast import _is_iast_enabled
-from ddtrace.appsec._iast._iast_request_context import in_iast_context
+from ddtrace.appsec._iast._iast_request_context import get_iast_stacktrace_reported
+from ddtrace.appsec._iast._iast_request_context import set_iast_stacktrace_reported
 from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
 from ddtrace.appsec._iast._patch import _iast_instrument_starlette_request
 from ddtrace.appsec._iast._patch import _iast_instrument_starlette_request_body
@@ -56,7 +57,7 @@ def _on_set_http_meta_iast(
 
 def _on_request_init(wrapped, instance, args, kwargs):
     wrapped(*args, **kwargs)
-    if _is_iast_enabled() and in_iast_context():
+    if _is_iast_enabled() and is_iast_request_enabled():
         try:
             instance.query_string = taint_pyobject(
                 pyobject=instance.query_string,
@@ -105,9 +106,6 @@ def _on_flask_patch(flask_version):
         _set_metric_iast_instrumented_source(OriginType.PATH)
         _set_metric_iast_instrumented_source(OriginType.QUERY)
 
-        # Instrumented on _ddtrace.appsec._asm_request_context._on_wrapped_view
-        _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER)
-
         try_wrap_function_wrapper(
             "werkzeug.wrappers.request",
             "Request.get_data",
@@ -129,9 +127,17 @@ def _on_flask_patch(flask_version):
             )
             _set_metric_iast_instrumented_source(OriginType.QUERY)
 
+        # Instrumented on _ddtrace.appsec._asm_request_context._on_wrapped_view
+        _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER)
+
+        # Instrumented on _on_set_request_tags_iast
+        _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME)
+        _set_metric_iast_instrumented_source(OriginType.COOKIE)
+        _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
+
 
 def _on_wsgi_environ(wrapped, _instance, args, kwargs):
-    if _is_iast_enabled() and args and in_iast_context():
+    if _is_iast_enabled() and args and is_iast_request_enabled():
         return wrapped(*((taint_structure(args[0], OriginType.HEADER_NAME, OriginType.HEADER),) + args[1:]), **kwargs)
 
     return wrapped(*args, **kwargs)
@@ -140,6 +146,13 @@ def _on_wsgi_environ(wrapped, _instance, args, kwargs):
 def _on_django_patch():
     if _is_iast_enabled():
         try:
+            when_imported("django.http.request")(
+                lambda m: try_wrap_function_wrapper(
+                    m,
+                    "QueryDict.__getitem__",
+                    functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
+                )
+            )
             # we instrument those sources on _on_django_func_wrapped
             _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
             _set_metric_iast_instrumented_source(OriginType.HEADER)
@@ -150,13 +163,7 @@ def _on_django_patch():
             _set_metric_iast_instrumented_source(OriginType.PARAMETER)
             _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
             _set_metric_iast_instrumented_source(OriginType.BODY)
-            when_imported("django.http.request")(
-                lambda m: try_wrap_function_wrapper(
-                    m,
-                    "QueryDict.__getitem__",
-                    functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
-                )
-            )
+
         except Exception:
             log.debug("Unexpected exception while patch IAST functions", exc_info=True)
 
@@ -165,7 +172,7 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_):
     # If IAST is enabled, and we're wrapping a Django view call, taint the kwargs (view's
     # path parameters)
     if _is_iast_enabled() and fn_args and isinstance(fn_args[0], first_arg_expected_type):
-        if not in_iast_context():
+        if not is_iast_request_enabled():
             return
 
         http_req = fn_args[0]
@@ -278,18 +285,16 @@ def _on_grpc_response(message):
 
 
 def if_iast_taint_yield_tuple_for(origins, wrapped, instance, args, kwargs):
-    if _is_iast_enabled():
-        if not is_iast_request_enabled():
-            for key, value in wrapped(*args, **kwargs):
-                yield key, value
-        else:
+    if _is_iast_enabled() and is_iast_request_enabled():
+        try:
             for key, value in wrapped(*args, **kwargs):
                 new_key = taint_pyobject(pyobject=key, source_name=key, source_value=key, source_origin=origins[0])
                 new_value = taint_pyobject(
                     pyobject=value, source_name=key, source_value=value, source_origin=origins[1]
                 )
                 yield new_key, new_value
-
+        except Exception:
+            log.debug("Unexpected exception while tainting pyobject", exc_info=True)
     else:
         for key, value in wrapped(*args, **kwargs):
             yield key, value
@@ -319,7 +324,7 @@ def if_iast_taint_starlette_datastructures(origin, wrapped, instance, args, kwar
                     res.append(
                         taint_pyobject(
                             pyobject=element,
-                            source_name=origin_to_str(origin),
+                            source_name=element,
                             source_value=element,
                             source_origin=origin,
                         )
@@ -417,14 +422,7 @@ def _on_pre_tracedrequest_iast(ctx):
 
 
 def _on_set_request_tags_iast(request, span, flask_config):
-    if _is_iast_enabled():
-        _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME)
-        _set_metric_iast_instrumented_source(OriginType.COOKIE)
-        _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
-
-        if not is_iast_request_enabled():
-            return
-
+    if _is_iast_enabled() and is_iast_request_enabled():
         request.cookies = taint_structure(
             request.cookies,
             OriginType.COOKIE_NAME,
@@ -445,3 +443,69 @@ def _on_set_request_tags_iast(request, span, flask_config):
             OriginType.PARAMETER,
             override_pyobject_tainted=True,
         )
+
+
+def _on_django_finalize_response_pre(ctx, after_request_tags, request, response):
+    if not response or not _is_iast_enabled() or not is_iast_request_enabled() or get_iast_stacktrace_reported():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        content = response.content.decode("utf-8", errors="ignore")
+        asm_check_stacktrace_leak(content)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)
+
+
+def _on_django_technical_500_response(request, response, exc_type, exc_value, tb):
+    if not exc_value or not _is_iast_enabled() or not is_iast_request_enabled():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_report_stacktrace_leak_from_django_debug_page
+
+        exc_name = exc_type.__name__
+        module = tb.tb_frame.f_globals.get("__name__", "")
+        asm_report_stacktrace_leak_from_django_debug_page(exc_name, module)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak on 500 response view", exc_info=True)
+
+
+def _on_flask_finalize_request_post(response, _):
+    if not response or not _is_iast_enabled() or not is_iast_request_enabled() or get_iast_stacktrace_reported():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        content = response[0].decode("utf-8", errors="ignore")
+        asm_check_stacktrace_leak(content)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)
+
+
+def _on_asgi_finalize_response(body, _):
+    if not body or not _is_iast_enabled() or not is_iast_request_enabled():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        content = body.decode("utf-8", errors="ignore")
+        asm_check_stacktrace_leak(content)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)
+
+
+def _on_werkzeug_render_debugger_html(html):
+    if not html or not _is_iast_enabled() or not is_iast_request_enabled():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        asm_check_stacktrace_leak(html)
+        set_iast_stacktrace_reported(True)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)

ddtrace/appsec/_iast/_iast_request_context.py

@@ -46,6 +46,7 @@ def __init__(self, span: Optional[Span] = None):
         self.iast_reporter: Optional[IastSpanReporter] = None
         self.iast_span_metrics: Dict[str, int] = {}
         self.iast_stack_trace_id: int = 0
+        self.iast_stack_trace_reported: bool = False
 
 
 def _get_iast_context() -> Optional[IASTEnvironment]:
@@ -88,6 +89,19 @@ def get_iast_reporter() -> Optional[IastSpanReporter]:
     return None
 
 
+def get_iast_stacktrace_reported() -> bool:
+    env = _get_iast_context()
+    if env:
+        return env.iast_stack_trace_reported
+    return False
+
+
+def set_iast_stacktrace_reported(reported: bool) -> None:
+    env = _get_iast_context()
+    if env:
+        env.iast_stack_trace_reported = reported
+
+
 def get_iast_stacktrace_id() -> int:
     env = _get_iast_context()
     if env:

ddtrace/appsec/_iast/_listener.py

@@ -1,11 +1,16 @@
+from ddtrace.appsec._iast._handlers import _on_asgi_finalize_response
+from ddtrace.appsec._iast._handlers import _on_django_finalize_response_pre
 from ddtrace.appsec._iast._handlers import _on_django_func_wrapped
 from ddtrace.appsec._iast._handlers import _on_django_patch
+from ddtrace.appsec._iast._handlers import _on_django_technical_500_response
+from ddtrace.appsec._iast._handlers import _on_flask_finalize_request_post
 from ddtrace.appsec._iast._handlers import _on_flask_patch
 from ddtrace.appsec._iast._handlers import _on_grpc_response
 from ddtrace.appsec._iast._handlers import _on_pre_tracedrequest_iast
 from ddtrace.appsec._iast._handlers import _on_request_init
 from ddtrace.appsec._iast._handlers import _on_set_http_meta_iast
 from ddtrace.appsec._iast._handlers import _on_set_request_tags_iast
+from ddtrace.appsec._iast._handlers import _on_werkzeug_render_debugger_html
 from ddtrace.appsec._iast._handlers import _on_wsgi_environ
 from ddtrace.appsec._iast._iast_request_context import _iast_end_request
 from ddtrace.internal import core
@@ -18,11 +23,16 @@ def iast_listen():
     core.on("set_http_meta_for_asm", _on_set_http_meta_iast)
     core.on("django.patch", _on_django_patch)
     core.on("django.wsgi_environ", _on_wsgi_environ, "wrapped_result")
+    core.on("django.finalize_response.pre", _on_django_finalize_response_pre)
     core.on("django.func.wrapped", _on_django_func_wrapped)
+    core.on("django.technical_500_response", _on_django_technical_500_response)
     core.on("flask.patch", _on_flask_patch)
     core.on("flask.request_init", _on_request_init)
     core.on("flask._patched_request", _on_pre_tracedrequest_iast)
     core.on("flask.set_request_tags", _on_set_request_tags_iast)
+    core.on("flask.finalize_request.post", _on_flask_finalize_request_post)
+    core.on("asgi.finalize_response", _on_asgi_finalize_response)
+    core.on("werkzeug.render_debugger_html", _on_werkzeug_render_debugger_html)
 
     core.on("context.ended.wsgi.__call__", _iast_end_request)
     core.on("context.ended.asgi.__call__", _iast_end_request)