diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 879365fe13c..00000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,29 +0,0 @@ - - -### Summary of problem - -### Which version of dd-trace-py are you using? - -### Which version of pip are you using? - - -### Which libraries and their versions are you using? - -
- `pip freeze` - -
- -### How can we reproduce your problem? - -### What is the result that you get? - -### What is the result that you expected? diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 00000000000..1870fa32021 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,64 @@ +name: "Bug Report (Low Priority)" +description: "Create a public Bug Report. Note that these may not be addressed as quickly as the helpdesk and that looking up account information will be difficult." +title: "[BUG]: " +labels: bug +body: + - type: input + attributes: + label: Tracer Version(s) + description: "Version(s) of the tracer affected by this bug" + placeholder: "1.23.4, 2.8.0" + validations: + required: true + + - type: input + attributes: + label: Python Version(s) + description: "Version(s) of Python (`python --version`) that you've encountered this bug with" + placeholder: "Python 3.9.15" + validations: + required: true + + - type: input + attributes: + label: Pip Version(s) + description: "Version(s) of Pip (`pip --version`) that you've encountered this bug with" + placeholder: "pip 22.0.4" + validations: + required: true + + - type: textarea + attributes: + label: Bug Report + description: Please add a clear and concise description of the bug here + validations: + required: true + + - type: textarea + attributes: + label: Reproduction Code + description: Please add code here to help us reproduce the problem + validations: + required: false + + - type: textarea + attributes: + label: Error Logs + description: "Please provide any error logs from the tracer (`DD_TRACE_DEBUG=true` can help)" + validations: + required: false + + - type: textarea + attributes: + label: Libraries in Use + description: "Which libraries and their versions are you using? Paste output from `pip freeze` here." + validations: + required: false + + - type: input + attributes: + label: Operating System + description: "Provide your operating system and version (e.g. `uname -a`)" + placeholder: Darwin Kernel Version 23.6.0 + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..8c090d6d6d1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,9 @@ +blank_issues_enabled: false +contact_links: + - name: Bug Report (High Priority) + url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:python + about: Create an expedited Bug Report via the helpdesk (no login required). This will allow us to look up your account and allows you to provide additional information in private. Please do not create a GitHub issue to report a bug. + - name: Feature Request (High Priority) + url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:python&tf_1260825272270=pt_apm_category_feature_request + about: Create an expedited Feature Request via the helpdesk (no login required). This helps with prioritization and allows you to provide additional information in private. Please do not create a GitHub issue to request a feature. + diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 00000000000..736852848cf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,50 @@ +name: Feature Request (Low Priority) +description: Create a public Feature Request. Note that these may not be addressed as quickly as the helpdesk and that looking up account information will be difficult. +title: "[FEATURE]: " +labels: feature-request +body: + - type: input + attributes: + label: Package Name + description: "If your feature request is to add instrumentation support for a package please provide the name here" + placeholder: mysql + validations: + required: false + + - type: input + attributes: + label: Package Version(s) + description: "If your feature request is to add instrumentation support for a package please provide the version you use" + placeholder: 0.0.3 + validations: + required: false + + - type: textarea + attributes: + label: Describe the goal of the feature + description: A clear and concise goal of what you want to happen. + validations: + required: true + + - type: textarea + attributes: + label: Is your feature request related to a problem? + description: | + Please add a clear and concise description of your problem. + E.g. I'm unable to instrument my database queries... + validations: + required: false + + - type: textarea + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered + validations: + required: false + + - type: textarea + attributes: + label: Additional context + description: Add any other context or screenshots about the feature request here + validations: + required: false diff --git a/.riot/requirements/16ae097.txt b/.riot/requirements/10f3c55.txt similarity index 59% rename from .riot/requirements/16ae097.txt rename to .riot/requirements/10f3c55.txt index 9b59b497e97..34a8e65f917 100644 --- a/.riot/requirements/16ae097.txt +++ b/.riot/requirements/10f3c55.txt @@ -2,23 +2,24 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/16ae097.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/10f3c55.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==24.4.2 automat==24.8.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 -coverage[toml]==7.6.1 -cryptography==43.0.0 +coverage[toml]==7.6.9 +cryptography==43.0.3 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -27,51 +28,51 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 -importlib-metadata==8.4.0 +idna==3.10 +importlib-metadata==8.5.0 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg==3.2.1 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg==3.2.3 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 -pytest-cov==5.0.0 +pyopenssl==24.3.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -tomli==2.0.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +tomli==2.2.1 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zipp==3.20.1 -zope-interface==7.0.3 +zeep==4.3.1 +zipp==3.21.0 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.6.0 diff --git a/.riot/requirements/14ebf3b.txt b/.riot/requirements/14ebf3b.txt index c762c1a65d3..a1e350a58fc 100644 --- a/.riot/requirements/14ebf3b.txt +++ b/.riot/requirements/14ebf3b.txt @@ -4,20 +4,20 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/14ebf3b.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 exceptiongroup==1.2.2 hypothesis==6.45.0 iniconfig==2.0.0 -mako==1.3.5 -markupsafe==2.1.5 +mako==1.3.8 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 -tomli==2.0.1 +tomli==2.2.1 diff --git a/.riot/requirements/2848d2c.txt b/.riot/requirements/1532dd6.txt similarity index 83% rename from .riot/requirements/2848d2c.txt rename to .riot/requirements/1532dd6.txt index fbd5a04bf84..bb2d4670425 100644 --- a/.riot/requirements/2848d2c.txt +++ b/.riot/requirements/1532dd6.txt @@ -2,22 +2,23 @@ # This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/2848d2c.in +# pip-compile --allow-unsafe --no-annotate --resolver=backtracking .riot/requirements/1532dd6.in # arrow==1.2.3 asgiref==3.7.2 attrs==24.2.0 autobahn==23.1.2 automat==22.10.0 +bcrypt==4.2.1 blessed==1.20.0 cached-property==1.5.2 -certifi==2024.8.30 +certifi==2024.12.14 cffi==1.15.1 channels==3.0.5 -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 constantly==15.1.0 coverage[toml]==7.2.7 -cryptography==43.0.0 +cryptography==44.0.0 daphne==3.0.2 django==3.2.25 django-configurations==2.4.2 @@ -28,11 +29,11 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 +idna==3.10 importlib-metadata==6.7.0 incremental==22.10.0 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 @@ -44,7 +45,7 @@ pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pylibmc==1.6.3 -pyopenssl==24.2.1 +pyopenssl==24.3.0 pytest==7.4.4 pytest-cov==4.1.0 pytest-django[testing]==3.10.0 @@ -52,13 +53,13 @@ pytest-mock==3.11.1 pytest-randomly==3.12.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.31.0 requests-file==2.1.0 requests-toolbelt==1.0.0 service-identity==21.1.0 -six==1.16.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 sqlparse==0.4.4 diff --git a/.riot/requirements/1ef9f39.txt b/.riot/requirements/1655cb3.txt similarity index 60% rename from .riot/requirements/1ef9f39.txt rename to .riot/requirements/1655cb3.txt index 024457a7bfe..f2ceb6dcf0f 100644 --- a/.riot/requirements/1ef9f39.txt +++ b/.riot/requirements/1655cb3.txt @@ -2,23 +2,24 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ef9f39.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1655cb3.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==24.4.2 automat==24.8.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 -coverage[toml]==7.6.1 -cryptography==43.0.0 +coverage[toml]==7.6.9 +cryptography==44.0.0 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -26,48 +27,48 @@ django-q==1.3.6 django-redis==4.5.0 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 +idna==3.10 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg==3.2.1 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg==3.2.3 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 -pytest-cov==5.0.0 +pyopenssl==24.3.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zope-interface==7.0.3 +zeep==4.3.1 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.6.0 diff --git a/.riot/requirements/175d0d6.txt b/.riot/requirements/175d0d6.txt index 2bdf82d04bc..443e7c0ee18 100644 --- a/.riot/requirements/175d0d6.txt +++ b/.riot/requirements/175d0d6.txt @@ -4,18 +4,18 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/175d0d6.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 hypothesis==6.45.0 iniconfig==2.0.0 -mako==1.3.5 -markupsafe==2.1.5 +mako==1.3.8 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 diff --git a/.riot/requirements/19255aa.txt b/.riot/requirements/19255aa.txt index 76a9c7370b5..f8375d7e478 100644 --- a/.riot/requirements/19255aa.txt +++ b/.riot/requirements/19255aa.txt @@ -4,20 +4,20 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/19255aa.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 exceptiongroup==1.2.2 hypothesis==6.45.0 iniconfig==2.0.0 mako==1.1.6 -markupsafe==2.1.5 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 -tomli==2.0.1 +tomli==2.2.1 diff --git a/.riot/requirements/d5fcd88.txt b/.riot/requirements/1aa41b2.txt similarity index 83% rename from .riot/requirements/d5fcd88.txt rename to .riot/requirements/1aa41b2.txt index acdaba91c5a..79322e51fe1 100644 --- a/.riot/requirements/d5fcd88.txt +++ b/.riot/requirements/1aa41b2.txt @@ -2,22 +2,23 @@ # This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/d5fcd88.in +# pip-compile --allow-unsafe --no-annotate --resolver=backtracking .riot/requirements/1aa41b2.in # arrow==1.2.3 asgiref==3.7.2 attrs==24.2.0 autobahn==23.1.2 automat==22.10.0 +bcrypt==4.2.1 blessed==1.20.0 cached-property==1.5.2 -certifi==2024.8.30 +certifi==2024.12.14 cffi==1.15.1 channels==4.0.0 -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 constantly==15.1.0 coverage[toml]==7.2.7 -cryptography==43.0.0 +cryptography==44.0.0 daphne==4.0.0 django==3.2.25 django-configurations==2.4.2 @@ -28,11 +29,11 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 +idna==3.10 importlib-metadata==6.7.0 incremental==22.10.0 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 @@ -44,7 +45,7 @@ pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pylibmc==1.6.3 -pyopenssl==24.2.1 +pyopenssl==24.3.0 pytest==7.4.4 pytest-cov==4.1.0 pytest-django[testing]==3.10.0 @@ -52,13 +53,13 @@ pytest-mock==3.11.1 pytest-randomly==3.12.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.31.0 requests-file==2.1.0 requests-toolbelt==1.0.0 service-identity==21.1.0 -six==1.16.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 sqlparse==0.4.4 diff --git a/.riot/requirements/1b16023.txt b/.riot/requirements/1b16023.txt index 3ecfcd6426c..805c3b3f36e 100644 --- a/.riot/requirements/1b16023.txt +++ b/.riot/requirements/1b16023.txt @@ -4,22 +4,22 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/1b16023.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 exceptiongroup==1.2.2 hypothesis==6.45.0 importlib-metadata==8.5.0 iniconfig==2.0.0 mako==1.1.6 -markupsafe==2.1.5 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 -tomli==2.0.1 -zipp==3.20.2 +tomli==2.2.1 +zipp==3.21.0 diff --git a/.riot/requirements/1b19707.txt b/.riot/requirements/1b19707.txt index bb31e7d6d71..5a50cb0f571 100644 --- a/.riot/requirements/1b19707.txt +++ b/.riot/requirements/1b19707.txt @@ -4,22 +4,22 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/1b19707.in # -attrs==24.2.0 +attrs==24.3.0 coverage[toml]==7.6.1 exceptiongroup==1.2.2 hypothesis==6.45.0 importlib-metadata==8.5.0 iniconfig==2.0.0 -mako==1.3.5 +mako==1.3.8 markupsafe==2.1.5 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 +pytest==8.3.4 pytest-cov==5.0.0 pytest-mock==3.14.0 pytest-randomly==3.15.0 sortedcontainers==2.4.0 -tomli==2.0.1 +tomli==2.2.1 zipp==3.20.2 diff --git a/.riot/requirements/1b8d922.txt b/.riot/requirements/1b8d922.txt index 76a225cb035..5b8e1001d48 100644 --- a/.riot/requirements/1b8d922.txt +++ b/.riot/requirements/1b8d922.txt @@ -4,18 +4,18 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/1b8d922.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 hypothesis==6.45.0 iniconfig==2.0.0 mako==1.1.6 -markupsafe==2.1.5 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 diff --git a/.riot/requirements/d57247c.txt b/.riot/requirements/1c5581b.txt similarity index 59% rename from .riot/requirements/d57247c.txt rename to .riot/requirements/1c5581b.txt index ca0162432b9..4886bf012bf 100644 --- a/.riot/requirements/d57247c.txt +++ b/.riot/requirements/1c5581b.txt @@ -2,23 +2,24 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/d57247c.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1c5581b.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==24.4.2 automat==24.8.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 -coverage[toml]==7.6.1 -cryptography==43.0.0 +coverage[toml]==7.6.9 +cryptography==43.0.3 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -27,50 +28,50 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 -importlib-metadata==8.4.0 +idna==3.10 +importlib-metadata==8.5.0 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 -pytest-cov==5.0.0 +pyopenssl==24.3.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -tomli==2.0.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +tomli==2.2.1 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zipp==3.20.1 -zope-interface==7.0.3 +zeep==4.3.1 +zipp==3.21.0 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.6.0 diff --git a/.riot/requirements/1d5ebaf.txt b/.riot/requirements/1d5ebaf.txt index be7d1a0c0ef..782c31d7836 100644 --- a/.riot/requirements/1d5ebaf.txt +++ b/.riot/requirements/1d5ebaf.txt @@ -4,18 +4,18 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/1d5ebaf.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 hypothesis==6.45.0 iniconfig==2.0.0 mako==1.1.6 -markupsafe==2.1.5 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 diff --git a/.riot/requirements/1e53fef.txt b/.riot/requirements/1e53fef.txt index 6908416793c..96e109a75d7 100644 --- a/.riot/requirements/1e53fef.txt +++ b/.riot/requirements/1e53fef.txt @@ -4,22 +4,22 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/1e53fef.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 exceptiongroup==1.2.2 hypothesis==6.45.0 importlib-metadata==8.5.0 iniconfig==2.0.0 -mako==1.3.5 -markupsafe==2.1.5 +mako==1.3.8 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 -tomli==2.0.1 -zipp==3.20.2 +tomli==2.2.1 +zipp==3.21.0 diff --git a/.riot/requirements/1d3001d.txt b/.riot/requirements/1f06d17.txt similarity index 60% rename from .riot/requirements/1d3001d.txt rename to .riot/requirements/1f06d17.txt index 92cb90787bc..b22a36b05b9 100644 --- a/.riot/requirements/1d3001d.txt +++ b/.riot/requirements/1f06d17.txt @@ -2,23 +2,24 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1d3001d.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1f06d17.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==24.4.2 automat==24.8.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 -coverage[toml]==7.6.1 -cryptography==43.0.0 +coverage[toml]==7.6.9 +cryptography==44.0.0 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -26,48 +27,48 @@ django-q==1.3.6 django-redis==4.5.0 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 +idna==3.10 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg==3.2.1 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg==3.2.3 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 -pytest-cov==5.0.0 +pyopenssl==24.3.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zope-interface==7.0.3 +zeep==4.3.1 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.6.0 diff --git a/.riot/requirements/27d0ff8.txt b/.riot/requirements/27d0ff8.txt index 291fe50cacc..aae68a87555 100644 --- a/.riot/requirements/27d0ff8.txt +++ b/.riot/requirements/27d0ff8.txt @@ -4,18 +4,18 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/27d0ff8.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 hypothesis==6.45.0 iniconfig==2.0.0 -mako==1.3.5 -markupsafe==2.1.5 +mako==1.3.8 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 diff --git a/.riot/requirements/1fe2c8e.txt b/.riot/requirements/4de03a5.txt similarity index 64% rename from .riot/requirements/1fe2c8e.txt rename to .riot/requirements/4de03a5.txt index 60fa0418337..5f1cc3a70e4 100644 --- a/.riot/requirements/1fe2c8e.txt +++ b/.riot/requirements/4de03a5.txt @@ -2,24 +2,25 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1fe2c8e.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/4de03a5.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==23.1.2 automat==24.8.1 backports-zoneinfo==0.2.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 coverage[toml]==7.6.1 -cryptography==43.0.0 +cryptography==44.0.0 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -28,50 +29,50 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 -importlib-metadata==8.4.0 +idna==3.10 +importlib-metadata==8.5.0 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 +pyopenssl==24.3.0 +pytest==8.3.4 pytest-cov==5.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -tomli==2.0.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +tomli==2.2.1 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zipp==3.20.1 -zope-interface==7.0.3 +zeep==4.3.1 +zipp==3.20.2 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.3.0 diff --git a/.riot/requirements/cbc433f.txt b/.riot/requirements/556d901.txt similarity index 60% rename from .riot/requirements/cbc433f.txt rename to .riot/requirements/556d901.txt index 8dfa49d4a04..7c70525020c 100644 --- a/.riot/requirements/cbc433f.txt +++ b/.riot/requirements/556d901.txt @@ -2,23 +2,24 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/cbc433f.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/556d901.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==24.4.2 automat==24.8.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 -coverage[toml]==7.6.1 -cryptography==43.0.0 +coverage[toml]==7.6.9 +cryptography==44.0.0 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -27,49 +28,49 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 +idna==3.10 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg==3.2.1 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg==3.2.3 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 -pytest-cov==5.0.0 +pyopenssl==24.3.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -tomli==2.0.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +tomli==2.2.1 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zope-interface==7.0.3 +zeep==4.3.1 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.6.0 diff --git a/.riot/requirements/b48f841.txt b/.riot/requirements/b48f841.txt index 16377236e8d..8143a485259 100644 --- a/.riot/requirements/b48f841.txt +++ b/.riot/requirements/b48f841.txt @@ -4,18 +4,18 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/b48f841.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 hypothesis==6.45.0 iniconfig==2.0.0 mako==1.1.6 -markupsafe==2.1.5 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 diff --git a/.riot/requirements/11065bb.txt b/.riot/requirements/b6e9905.txt similarity index 64% rename from .riot/requirements/11065bb.txt rename to .riot/requirements/b6e9905.txt index d93c2d7bd31..c17865f1eae 100644 --- a/.riot/requirements/11065bb.txt +++ b/.riot/requirements/b6e9905.txt @@ -2,24 +2,25 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/11065bb.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/b6e9905.in # arrow==1.3.0 asgiref==3.8.1 -attrs==24.2.0 +attrs==24.3.0 autobahn==23.1.2 automat==24.8.1 backports-zoneinfo==0.2.1 +bcrypt==4.2.1 blessed==1.20.0 -certifi==2024.8.30 -cffi==1.17.0 -channels==4.1.0 -charset-normalizer==3.3.2 +certifi==2024.12.14 +cffi==1.17.1 +channels==4.2.0 +charset-normalizer==3.4.0 constantly==23.10.4 coverage[toml]==7.6.1 -cryptography==43.0.0 +cryptography==44.0.0 daphne==4.1.2 -django==4.2.15 +django==4.2.17 django-configurations==2.5.1 django-picklefield==3.2 django-pylibmc==0.6.1 @@ -28,51 +29,51 @@ django-redis==4.5.0 exceptiongroup==1.2.2 hyperlink==21.0.0 hypothesis==6.45.0 -idna==3.8 -importlib-metadata==8.4.0 +idna==3.10 +importlib-metadata==8.5.0 incremental==24.7.2 iniconfig==2.0.0 -isodate==0.6.1 +isodate==0.7.2 lxml==5.3.0 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 -platformdirs==4.2.2 +packaging==24.2 +platformdirs==4.3.6 pluggy==1.5.0 -psycopg==3.2.1 -psycopg2-binary==2.9.9 -pyasn1==0.6.0 -pyasn1-modules==0.4.0 +psycopg==3.2.3 +psycopg2-binary==2.9.10 +pyasn1==0.6.1 +pyasn1-modules==0.4.1 pycparser==2.22 pylibmc==1.6.3 -pyopenssl==24.2.1 -pytest==8.3.2 +pyopenssl==24.3.0 +pytest==8.3.4 pytest-cov==5.0.0 pytest-django[testing]==3.10.0 pytest-mock==3.14.0 pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 python-memcached==1.62 -pytz==2024.1 +pytz==2024.2 redis==2.10.6 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 -service-identity==24.1.0 -six==1.16.0 +service-identity==24.2.0 +six==1.17.0 sortedcontainers==2.4.0 spyne==2.14.0 -sqlparse==0.5.1 -tomli==2.0.1 -twisted[tls]==24.7.0 +sqlparse==0.5.3 +tomli==2.2.1 +twisted[tls]==24.11.0 txaio==23.1.1 -types-python-dateutil==2.9.0.20240821 +types-python-dateutil==2.9.0.20241206 typing-extensions==4.12.2 -urllib3==2.2.2 +urllib3==2.2.3 wcwidth==0.2.13 -zeep==4.2.1 -zipp==3.20.1 -zope-interface==7.0.3 +zeep==4.3.1 +zipp==3.20.2 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==74.0.0 +setuptools==75.3.0 diff --git a/.riot/requirements/e8d8aa5.txt b/.riot/requirements/e8d8aa5.txt index 32eaaf1e0a5..54a8f705f5c 100644 --- a/.riot/requirements/e8d8aa5.txt +++ b/.riot/requirements/e8d8aa5.txt @@ -4,18 +4,18 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/e8d8aa5.in # -attrs==24.2.0 -coverage[toml]==7.6.1 +attrs==24.3.0 +coverage[toml]==7.6.9 hypothesis==6.45.0 iniconfig==2.0.0 -mako==1.3.5 -markupsafe==2.1.5 +mako==1.3.8 +markupsafe==3.0.2 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 -pytest-cov==5.0.0 +pytest==8.3.4 +pytest-cov==6.0.0 pytest-mock==3.14.0 -pytest-randomly==3.15.0 +pytest-randomly==3.16.0 sortedcontainers==2.4.0 diff --git a/.riot/requirements/eda6d79.txt b/.riot/requirements/eda6d79.txt index fd1ab1bf1d4..1cd10ab27f3 100644 --- a/.riot/requirements/eda6d79.txt +++ b/.riot/requirements/eda6d79.txt @@ -4,7 +4,7 @@ # # pip-compile --allow-unsafe --no-annotate .riot/requirements/eda6d79.in # -attrs==24.2.0 +attrs==24.3.0 coverage[toml]==7.6.1 exceptiongroup==1.2.2 hypothesis==6.45.0 @@ -14,12 +14,12 @@ mako==1.1.6 markupsafe==2.1.5 mock==5.1.0 opentracing==2.4.0 -packaging==24.1 +packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 +pytest==8.3.4 pytest-cov==5.0.0 pytest-mock==3.14.0 pytest-randomly==3.15.0 sortedcontainers==2.4.0 -tomli==2.0.1 +tomli==2.2.1 zipp==3.20.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d259f8624a..2039d597f9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,164 @@ Changelogs for versions not listed here can be found at https://github.com/DataDog/dd-trace-py/releases +--- + +## 2.18.1 + + +### Bug Fixes + +Profiling: +- Fixes an issue where the memory allocation profiler can cause a segmentation fault due to data races when accessing its own global data structures from multiple threads. +- Fixes a bug where profiling mutexes were not cleared on fork in the child process. This could cause deadlocks in certain configurations. + +Tracing: +- celery: Fixes an issue where `celery.apply` spans from Celery prerun got closed too soon leading to span tags being missing. + +--- + +## 2.18.0 + +### Upgrade Notes +- ASM + - With this upgrade, you can now control how the stack trace report are cropped when reported for exploit prevention or IAST. + - `DD_APPSEC_MAX_STACK_TRACE_DEPTH` allowed to control the maximum stack trace size reported (default 32) + - `DD_APPSEC_MAX_STACK_TRACE_DEPTH_TOP_PERCENT` allows now to specify how the stack trace is cropped as a percentage. + + For example, a value of 100 will report the top DD_APPSEC_MAX_STACK_TRACE_DEPTH frames from the stack, while a value of 0 will report the bottom DD_APPSEC_MAX_STACK_TRACE_DEPTH frames of the trace. A value of 50 will report half of DD_APPSEC_MAX_STACK_TRACE_DEPTH (rounded down) frames from the top of the stack and the rest from bottom. Default value is 75. + - Upgrades `libddwaf` to 1.22.0 + - Upgrades `libddwaf` to 1.21.0 and security rule file to 1.13.3 + + +### Deprecation Notes +- Python 3.7 support is deprecated and will be removed in 3.0 + + +### New Features +- CI Visibility + - Beta release of the new version of the pytest plugin, introducing the following features: + - [Auto Test Retries](https://docs.datadoghq.com/tests/flaky_test_management/auto_test_retries) + - [Early Flake Detection](https://docs.datadoghq.com/tests/flaky_test_management/early_flake_detection) + - Improved coverage collection for [Test Impact Analysis](https://docs.datadoghq.com/tests/test_impact_analysis) (formerly Intelligent Test Runner) now uses an internal collection method instead of [coverage.py](https://github.com/nedbat/coveragepy), with improved dependency discovery. + + Set the `DD_PYTEST_USE_NEW_PLUGIN_BETA` environment variable to `true` to use this new version. + + **NOTE:** this new version of the plugin introduces breaking changes: + - `module`, `suite`, and `test` names are now parsed from the `item.nodeid` attribute + - test names now include the class for class-based tests + - Test skipping by Test Impact Analysis (formerly Intelligent Test Runner) is now done at the suite level, instead of at the test level +- Adds support for [Selenium and RUM integration](https://docs.datadoghq.com/tests/browser_tests/) + +- Code Security + - Introduces "Standalone Code Security", a feature that disables APM in the tracer but keeps Code Security (IAST) enabled. In order to enable it, set the environment variables `DD_IAST_ENABLED=1` and `DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED=1`. + +- LLM Observability + - Adds support to automatically submit Vertex AI Python calls to LLM Observability. + - `vertexai`: Introduces tracing support for Google's Vertex AI SDK for Python's `generate_content` and `send_message` calls. See [the docs](https://ddtrace.readthedocs.io/en/stable/integrations.html#vertexai) for more information. + +- Profiling + - Profiler uses agent url configured via `tracer.configure()` + + +### Bug Fixes +- ASM + - Ensures that common patches for exploit prevention and sca are only loaded if required, and only loaded once. + - Resolves an issue where AppSec was using a patched JSON loads, creating telemetry errors. + - Resolves an issue where some root span where not appropriately tagged for ASM standalone. + - ASM: Resolves an issue where AppSec was using a patched request and builtins functions, + creating telemetry errors. + +- CI Visibility + - Fixes an issue where the CIVisbility service would incorrectly default the tracer env to `None` in EVP proxy mode if `DD_ENV` was not specified but the agent had a default environment set to a value other than `none` (eg: using `DD_APM_ENV` in the agent's environment). + - Updates the inferred base service name algorithm to ensure that arguments following `--ddtrace` are no longer skipped when executing tests with pytest. Previously, the algorithm misinterpreted these arguments as standard flags, overlooking possible test paths that may contribute to the inferred service name. + +- Code Security + - Patches the module dir function so original pre-patch results are not changed. + - Resolves a patching issue with `psycopg3`. + - This fix resolves an issue where the modulo (%) operator would not be replaced correctly for bytes and bytesarray if IAST is enabled. + - Ensures IAST SSRF vulnerability redacts the url query parameters correctly. + - Adds `umap`, `numba` and `pynndescent` to the Code Security denylist. + +- Crashtracking + - Resolves issue where the crashtracker receiver may leave a zombie process behind after a crash. + +- Lib-Injection + - Ensures any user defined `sitecustomize.py` are preserved when auto-injecting. + - Supports Python 2.7+ for injection compatibility check. + - Resolves an issue where the default versions of `click` and `jinja2` installed on 3.8 were outside of the allowed minimum versions for autoinstrumentation. + +- LLM Observability + - Ensures bedrock spans are finished even when streamed responses are not fully consumed. + - `langchain`: Resolves a JSON decoding issue resulting from tagging streamed outputs from chains ending with a PydanticOutputParser. + - Fixes an issue where decorators were not tracing generator functions properly. + +- Profiling + - Updates setup.py to ignore int-ptr conversion warnings for the profiler stack.pyx file. This is important because gcc 14 makes these conversions an error, alpine 3.21.0 ships with gcc 14, and any patch version of a Python alpine image cut after December 5th, 2024, will have this issue. + - Fixes unbounded memory usage growth caused by keeping arbitrary user-generated strings (e.g. asyncio Task names) in an internal table and never removing them. + - Fixes an issue where `asyncio` task names are not properly propagated when using stack v2, i.e. when `DD_PROFILING_STACK_V2_ENABLED` is set. Fixes an issue where `asyncio` tasks are not associated with spans when using stack v2, i.e. when `DD_PROFILING_STACK_V2_ENABLED` is set. + +- Telemetry + - Ensures that Telemetry heartbeats are not skipped for forked processes, as doing so could result in the dependency list being lost over time. + +- Tracing + - `botocore`: This fix resolves an issue in the Bedrock integration where not consuming the full response stream would prevent spans from finishing. + - `botocore`: This fix resolves the issue where the span pointer for deserialized DynamoDB requests (through the resource-based API) were not being generated. + - `botocore`: This fix resolves an issue where our span pointer calculation code added recently logged unactionable messages. + - `celery`: This fix resolves two issues with context propagation in celery + 1. Invalid span parentage when task A calls task B async and task A errors out, causing A's queuing of B, and B itself to not be parented under A. + 2. Invalid context propagation from client to workers, and across retries, causing multiple traces instead of a single trace + - `celery`: Changes celery `out.host` span tag to point towards broker host url instead of local celery process hostname. Fixes inferred service representation issues when using celery. + - `grpcaio`: Resolves a concurrency bug where distributed tracing headers were overwritten resulting in spans being assigned to the wrong trace. + - `kafka`: Fixes an issue with Kafka consumer spans not using the active trace context when distributed tracing was enabled and no valid distributed context found was found within a consumed message. + + +### Other Changes +- Tracing + - Removed x-forwarded from headers used for client IP resolution (but not from collected headers). We lack evidence of actual usage, and whether this should follow RFC 7239 or regular XFF list format. + +--- + +## 2.17.3 + +### Bug Fixes + +- SCA: + - Ensure that Telemetry heartbeats are not skipped for forked processes, as doing so could result in the dependency list being lost over time. + +- Celery: + - This fix resolves two issues with context propagation in celery + - 1. Invalid span parentage when task A calls task B async and task A errors out, causing A's queuing of B, and B itself to not be parented under A. + - 2. Invalid context propagation from client to workers, and across retries, causing multiple traces instead of a single trace + +- Code Security: + - This fix resolves a patching issue with psycopg3. + - This fix resolves an issue where the modulo (%) operator would not be replaced correctly for bytes and bytesarray if IAST is enabled. + - Ensure IAST SSRF vulnerability redacts the url query parameters correctly. + +- Profiling: + - Updates setup.py to ignore int-ptr conversion warnings for the profiler stack.pyx file. This is important because gcc 14 makes these conversions an error, alpine 3.21.0 ships with gcc 14, and any patch version of a Python alpine image cut after December 5th, 2024, will have this issue. + +--- + +## 2.16.6 + +### Bug Fixes + +- SCA: + - Ensure that Telemetry heartbeats are not skipped for forked processes, as doing so could result in the dependency list being lost over time. + +- Code Security: + - Resolve a patching issue with psycopg3. + - Resolve an issue where the modulo (%) operator would not be replaced correctly for bytes and bytesarray if IAST is enabled. + - Ensure IAST SSRF vulnerability redacts the url query parameters correctly. + +- Lib-Injection: + - Fix injection guardrail check when sys.argv is not available. + +- Profiling + - Updates setup.py to ignore int-ptr conversion warnings for the profiler stack.pyx file. This is important because gcc 14 makes these conversions an error, alpine 3.21.0 ships with gcc 14, and any patch version of a Python alpine image cut after December 5th, 2024, will have this issue. + + --- ## 2.17.2 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..d57cdcb881a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +This document outlines the security policy for the Datadog Python client library (aka Python tracer) and what to do if you discover a security vulnerability in the project. +Most notably, please do not share the details in a public forum (such as in a discussion, issue, or pull request) but instead reach out to us with the details. +This gives us an opportunity to release a fix for others to benefit from by the time details are made public. + +## Supported Versions + +We accept vulnerability submissions for the [currently maintained release](https://github.com/DataDog/dd-trace-py/releases). + +## Reporting a Vulnerability + +If you discover a vulnerability in the Datadog Python client library (or any Datadog product for that matter) please submit details to the following email address: + +* [security@datadoghq.com](mailto:security@datadoghq.com) diff --git a/benchmarks/appsec_iast_aspects/scenario.py b/benchmarks/appsec_iast_aspects/scenario.py index 145b43f1633..26d9f2a37bd 100644 --- a/benchmarks/appsec_iast_aspects/scenario.py +++ b/benchmarks/appsec_iast_aspects/scenario.py @@ -11,8 +11,8 @@ from ddtrace.appsec._iast._iast_request_context import start_iast_context except ImportError: # Pre 2.15 - from ddtrace.appsec._iast._taint_tracking import create_context as start_iast_context - from ddtrace.appsec._iast._taint_tracking import reset_context as end_iast_context + from ddtrace.appsec._iast._taint_tracking._context import create_context as start_iast_context + from ddtrace.appsec._iast._taint_tracking._context import reset_context as end_iast_context set_iast_request_enabled = lambda x: None # noqa: E731 diff --git a/benchmarks/rate_limiter/scenario.py b/benchmarks/rate_limiter/scenario.py index 5c1f80f2537..5210647ef89 100644 --- a/benchmarks/rate_limiter/scenario.py +++ b/benchmarks/rate_limiter/scenario.py @@ -9,7 +9,8 @@ class RateLimiter(bm.Scenario): num_windows: int def run(self): - from ddtrace.internal.compat import time_ns + from time import time_ns + from ddtrace.internal.rate_limiter import RateLimiter rate_limiter = RateLimiter(rate_limit=self.rate_limit, time_window=self.time_window) diff --git a/ddtrace/__init__.py b/ddtrace/__init__.py index 1954c1961c9..1f2049cd0a5 100644 --- a/ddtrace/__init__.py +++ b/ddtrace/__init__.py @@ -33,6 +33,11 @@ from .version import get_version # noqa: E402 +# TODO(mabdinur): Remove this once we have a better way to start the mini agent +from ddtrace.internal.serverless.mini_agent import maybe_start_serverless_mini_agent as _start_mini_agent + +_start_mini_agent() + # DEV: Import deprecated tracer module in order to retain side-effect of package # initialization, which added this module to sys.modules. We catch deprecation # warnings as this is only to retain a side effect of the package diff --git a/ddtrace/_trace/span.py b/ddtrace/_trace/span.py index db90a769cd9..afb3496db80 100644 --- a/ddtrace/_trace/span.py +++ b/ddtrace/_trace/span.py @@ -1,6 +1,7 @@ import math import pprint import sys +from time import time_ns import traceback from types import TracebackType from typing import Any @@ -46,7 +47,6 @@ from ddtrace.internal.compat import StringIO from ddtrace.internal.compat import ensure_text from ddtrace.internal.compat import is_integer -from ddtrace.internal.compat import time_ns from ddtrace.internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS from ddtrace.internal.constants import SPAN_API_DATADOG from ddtrace.internal.logger import get_logger diff --git a/ddtrace/_trace/tracer.py b/ddtrace/_trace/tracer.py index 6027976d6dc..fa0c89cdd7f 100644 --- a/ddtrace/_trace/tracer.py +++ b/ddtrace/_trace/tracer.py @@ -53,7 +53,6 @@ from ddtrace.internal.serverless import in_aws_lambda from ddtrace.internal.serverless import in_azure_function from ddtrace.internal.serverless import in_gcp_function -from ddtrace.internal.serverless.mini_agent import maybe_start_serverless_mini_agent from ddtrace.internal.service import ServiceStatusError from ddtrace.internal.utils import _get_metas_to_propagate from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning @@ -211,8 +210,6 @@ def __init__( :param dogstatsd_url: The DogStatsD URL. """ - maybe_start_serverless_mini_agent() - self._filters: List[TraceFilter] = [] # globally set tags diff --git a/ddtrace/appsec/__init__.py b/ddtrace/appsec/__init__.py index bc89c0f2127..05d1a852710 100644 --- a/ddtrace/appsec/__init__.py +++ b/ddtrace/appsec/__init__.py @@ -18,7 +18,7 @@ def load_appsec(): def load_iast(): """Lazily load the iast module listeners.""" - from ddtrace.appsec._iast._iast_request_context import iast_listen + from ddtrace.appsec._iast._listener import iast_listen global _IAST_TO_BE_LOADED if _IAST_TO_BE_LOADED: diff --git a/ddtrace/appsec/_asm_request_context.py b/ddtrace/appsec/_asm_request_context.py index e3a87672e05..adb78a4447c 100644 --- a/ddtrace/appsec/_asm_request_context.py +++ b/ddtrace/appsec/_asm_request_context.py @@ -16,6 +16,12 @@ from ddtrace.appsec._constants import APPSEC from ddtrace.appsec._constants import EXPLOIT_PREVENTION from ddtrace.appsec._constants import SPAN_DATA_NAMES +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject +from ddtrace.appsec._iast._taint_utils import taint_structure +from ddtrace.appsec._iast._utils import _is_iast_enabled from ddtrace.appsec._utils import add_context_log from ddtrace.appsec._utils import get_triggers from ddtrace.internal import core @@ -488,13 +494,8 @@ def _on_wrapped_view(kwargs): return_value[0] = callback_block # If IAST is enabled, taint the Flask function kwargs (path parameters) - from ddtrace.appsec._iast._utils import _is_iast_enabled if _is_iast_enabled() and kwargs: - from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - if not is_iast_request_enabled(): return return_value @@ -511,11 +512,6 @@ def _on_set_request_tags(request, span, flask_config): from ddtrace.appsec._iast._utils import _is_iast_enabled if _is_iast_enabled(): - from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled - from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_utils import taint_structure - _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME) _set_metric_iast_instrumented_source(OriginType.COOKIE) diff --git a/ddtrace/appsec/_common_module_patches.py b/ddtrace/appsec/_common_module_patches.py index e7ce12d13e9..215d8b05ee6 100644 --- a/ddtrace/appsec/_common_module_patches.py +++ b/ddtrace/appsec/_common_module_patches.py @@ -14,6 +14,7 @@ import ddtrace from ddtrace.appsec._asm_request_context import get_blocked from ddtrace.appsec._constants import WAF_ACTIONS +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL from ddtrace.internal import core @@ -60,14 +61,12 @@ def wrapped_read_F3E51D71B4EC16EF(original_read_callable, instance, args, kwargs """ wrapper for _io.BytesIO and _io.StringIO read function """ - from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled - result = original_read_callable(*args, **kwargs) if asm_config._iast_enabled and is_iast_request_enabled(): from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import Source - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject ranges = get_tainted_ranges(instance) if len(ranges) > 0: @@ -89,8 +88,6 @@ def wrapped_open_CFDDB7ABBA9081B6(original_open_callable, instance, args, kwargs """ wrapper for open file function """ - from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled - if asm_config._iast_enabled and is_iast_request_enabled(): try: from ddtrace.appsec._iast.taint_sinks.path_traversal import check_and_report_path_traversal @@ -180,8 +177,6 @@ def wrapped_request_D8CB81E472AF98A2(original_request_callable, instance, args, wrapper for third party requests.request function https://requests.readthedocs.io """ - from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled - if asm_config._iast_enabled and is_iast_request_enabled(): from ddtrace.appsec._iast.taint_sinks.ssrf import _iast_report_ssrf @@ -222,8 +217,6 @@ def wrapped_system_5542593D237084A7(original_command_callable, instance, args, k """ command = args[0] if args else kwargs.get("command", None) if command is not None: - from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled - if asm_config._iast_enabled and is_iast_request_enabled(): from ddtrace.appsec._iast.taint_sinks.command_injection import _iast_report_cmdi diff --git a/ddtrace/appsec/_iast/_handlers.py b/ddtrace/appsec/_iast/_handlers.py index 4ba0ecc86e0..2c681e548e9 100644 --- a/ddtrace/appsec/_iast/_handlers.py +++ b/ddtrace/appsec/_iast/_handlers.py @@ -5,15 +5,22 @@ from wrapt import wrap_function_wrapper as _w from ddtrace.appsec._iast import _is_iast_enabled +from ddtrace.appsec._iast._iast_request_context import in_iast_context from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source from ddtrace.appsec._iast._patch import _iast_instrument_starlette_request from ddtrace.appsec._iast._patch import _iast_instrument_starlette_request_body from ddtrace.appsec._iast._patch import _iast_instrument_starlette_url from ddtrace.appsec._iast._patch import _patched_dictionary from ddtrace.appsec._iast._patch import try_wrap_function_wrapper +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking import origin_to_str +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_utils import taint_structure from ddtrace.internal.logger import get_logger +from ._iast_request_context import is_iast_request_enabled +from ._taint_tracking._taint_objects import taint_pyobject + MessageMapContainer = None try: @@ -48,15 +55,9 @@ def _on_set_http_meta_iast( def _on_request_init(wrapped, instance, args, kwargs): - from ddtrace.appsec._iast._iast_request_context import in_iast_context - wrapped(*args, **kwargs) if _is_iast_enabled() and in_iast_context(): try: - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import origin_to_str - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - instance.query_string = taint_pyobject( pyobject=instance.query_string, source_name=origin_to_str(OriginType.QUERY), @@ -75,8 +76,6 @@ def _on_request_init(wrapped, instance, args, kwargs): def _on_flask_patch(flask_version): if _is_iast_enabled(): - from ddtrace.appsec._iast._taint_tracking import OriginType - try_wrap_function_wrapper( "werkzeug.datastructures", "Headers.items", @@ -132,11 +131,7 @@ def _on_flask_patch(flask_version): def _on_wsgi_environ(wrapped, _instance, args, kwargs): - from ddtrace.appsec._iast._iast_request_context import in_iast_context - if _is_iast_enabled() and args and in_iast_context(): - from ddtrace.appsec._iast._taint_tracking import OriginType - return wrapped(*((taint_structure(args[0], OriginType.HEADER_NAME, OriginType.HEADER),) + args[1:]), **kwargs) return wrapped(*args, **kwargs) @@ -145,8 +140,6 @@ def _on_wsgi_environ(wrapped, _instance, args, kwargs): def _on_django_patch(): if _is_iast_enabled(): try: - from ddtrace.appsec._iast._taint_tracking import OriginType - # we instrument those sources on _on_django_func_wrapped _set_metric_iast_instrumented_source(OriginType.HEADER_NAME) _set_metric_iast_instrumented_source(OriginType.HEADER) @@ -169,15 +162,9 @@ def _on_django_patch(): def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_): - # If IAST is enabled and we're wrapping a Django view call, taint the kwargs (view's + # If IAST is enabled, and we're wrapping a Django view call, taint the kwargs (view's # path parameters) if _is_iast_enabled() and fn_args and isinstance(fn_args[0], first_arg_expected_type): - from ddtrace.appsec._iast._iast_request_context import in_iast_context - from ddtrace.appsec._iast._taint_tracking import OriginType # noqa: F401 - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted - from ddtrace.appsec._iast._taint_tracking import origin_to_str - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - if not in_iast_context(): return @@ -243,9 +230,6 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_): def _custom_protobuf_getattribute(self, name): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - ret = type(self).__saved_getattr(self, name) if isinstance(ret, (str, bytes, bytearray)): ret = taint_pyobject( @@ -295,9 +279,6 @@ def _on_grpc_response(message): def if_iast_taint_yield_tuple_for(origins, wrapped, instance, args, kwargs): if _is_iast_enabled(): - from ._iast_request_context import is_iast_request_enabled - from ._taint_tracking import taint_pyobject - if not is_iast_request_enabled(): for key, value in wrapped(*args, **kwargs): yield key, value @@ -316,17 +297,11 @@ def if_iast_taint_yield_tuple_for(origins, wrapped, instance, args, kwargs): def if_iast_taint_returned_object_for(origin, wrapped, instance, args, kwargs): value = wrapped(*args, **kwargs) - from ._iast_request_context import is_iast_request_enabled if _is_iast_enabled() and is_iast_request_enabled(): try: - from ._taint_tracking import is_pyobject_tainted - from ._taint_tracking import taint_pyobject - if not is_pyobject_tainted(value): name = str(args[0]) if len(args) else "http.request.body" - from ddtrace.appsec._iast._taint_tracking import OriginType - if origin == OriginType.HEADER and name.lower() in ["cookie", "cookies"]: origin = OriginType.COOKIE return taint_pyobject(pyobject=value, source_name=name, source_value=value, source_origin=origin) @@ -336,8 +311,6 @@ def if_iast_taint_returned_object_for(origin, wrapped, instance, args, kwargs): def _on_iast_fastapi_patch(): - from ddtrace.appsec._iast._taint_tracking import OriginType - # Cookies sources try_wrap_function_wrapper( "starlette.requests", diff --git a/ddtrace/appsec/_iast/_iast_request_context.py b/ddtrace/appsec/_iast/_iast_request_context.py index a28c2d3ff0d..07ad4c9c238 100644 --- a/ddtrace/appsec/_iast/_iast_request_context.py +++ b/ddtrace/appsec/_iast/_iast_request_context.py @@ -8,16 +8,11 @@ from ddtrace.appsec._constants import IAST from ddtrace.appsec._iast import _is_iast_enabled from ddtrace.appsec._iast import oce -from ddtrace.appsec._iast._handlers import _on_django_func_wrapped -from ddtrace.appsec._iast._handlers import _on_django_patch -from ddtrace.appsec._iast._handlers import _on_flask_patch -from ddtrace.appsec._iast._handlers import _on_grpc_response -from ddtrace.appsec._iast._handlers import _on_request_init -from ddtrace.appsec._iast._handlers import _on_set_http_meta_iast -from ddtrace.appsec._iast._handlers import _on_wsgi_environ from ddtrace.appsec._iast._metrics import _set_metric_iast_request_tainted from ddtrace.appsec._iast._metrics import _set_span_tag_iast_executed_sink from ddtrace.appsec._iast._metrics import _set_span_tag_iast_request_tainted +from ddtrace.appsec._iast._taint_tracking._context import create_context as create_propagation_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context as reset_propagation_context from ddtrace.appsec._iast.reporter import IastSpanReporter from ddtrace.constants import ORIGIN_KEY from ddtrace.internal import core @@ -63,15 +58,11 @@ def in_iast_context() -> bool: def start_iast_context(): if _is_iast_enabled(): - from ._taint_tracking import create_context as create_propagation_context - create_propagation_context() core.set_item(_IAST_CONTEXT, IASTEnvironment()) def end_iast_context(span: Optional[Span] = None): - from ._taint_tracking import reset_context as reset_propagation_context - env = _get_iast_context() if env is not None and env.span is span: finalize_iast_env(env) @@ -190,22 +181,3 @@ def _iast_start_request(span=None, *args, **kwargs): set_iast_request_enabled(request_iast_enabled) except Exception: log.debug("[IAST] Error starting IAST context", exc_info=True) - - -def _on_grpc_server_response(message): - _on_grpc_response(message) - - -def iast_listen(): - core.on("grpc.client.response.message", _on_grpc_response) - core.on("grpc.server.response.message", _on_grpc_server_response) - - core.on("set_http_meta_for_asm", _on_set_http_meta_iast) - core.on("django.patch", _on_django_patch) - core.on("django.wsgi_environ", _on_wsgi_environ, "wrapped_result") - core.on("django.func.wrapped", _on_django_func_wrapped) - core.on("flask.patch", _on_flask_patch) - core.on("flask.request_init", _on_request_init) - - core.on("context.ended.wsgi.__call__", _iast_end_request) - core.on("context.ended.asgi.__call__", _iast_end_request) diff --git a/ddtrace/appsec/_iast/_listener.py b/ddtrace/appsec/_iast/_listener.py new file mode 100644 index 00000000000..356199a3cad --- /dev/null +++ b/ddtrace/appsec/_iast/_listener.py @@ -0,0 +1,28 @@ +from ddtrace.appsec._iast._handlers import _on_django_func_wrapped +from ddtrace.appsec._iast._handlers import _on_django_patch +from ddtrace.appsec._iast._handlers import _on_flask_patch +from ddtrace.appsec._iast._handlers import _on_grpc_response +from ddtrace.appsec._iast._handlers import _on_request_init +from ddtrace.appsec._iast._handlers import _on_set_http_meta_iast +from ddtrace.appsec._iast._handlers import _on_wsgi_environ +from ddtrace.appsec._iast._iast_request_context import _iast_end_request +from ddtrace.internal import core + + +def iast_listen(): + core.on("grpc.client.response.message", _on_grpc_response) + core.on("grpc.server.response.message", _on_grpc_server_response) + + core.on("set_http_meta_for_asm", _on_set_http_meta_iast) + core.on("django.patch", _on_django_patch) + core.on("django.wsgi_environ", _on_wsgi_environ, "wrapped_result") + core.on("django.func.wrapped", _on_django_func_wrapped) + core.on("flask.patch", _on_flask_patch) + core.on("flask.request_init", _on_request_init) + + core.on("context.ended.wsgi.__call__", _iast_end_request) + core.on("context.ended.asgi.__call__", _iast_end_request) + + +def _on_grpc_server_response(message): + _on_grpc_response(message) diff --git a/ddtrace/appsec/_iast/_patch.py b/ddtrace/appsec/_iast/_patch.py index 92d776c79cb..b1bf1f04989 100644 --- a/ddtrace/appsec/_iast/_patch.py +++ b/ddtrace/appsec/_iast/_patch.py @@ -5,10 +5,12 @@ from wrapt import FunctionWrapper from ddtrace.appsec._common_module_patches import wrap_object +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking import origin_to_str +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject +from ddtrace.appsec._iast._taint_utils import taint_structure from ddtrace.internal.logger import get_logger -from ._taint_utils import taint_structure - log = get_logger(__name__) @@ -48,10 +50,6 @@ def _patched_dictionary(origin_key, origin_value, original_func, instance, args, def _iast_instrument_starlette_url(wrapped, instance, args, kwargs): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import origin_to_str - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - def path(self) -> str: return taint_pyobject( self.components.path, @@ -65,8 +63,6 @@ def path(self) -> str: def _iast_instrument_starlette_request(wrapped, instance, args, kwargs): - from ddtrace.appsec._iast._taint_tracking import OriginType - def receive(self): """This pattern comes from a Request._receive property, which returns a callable""" @@ -82,10 +78,6 @@ async def wrapped_property_call(): async def _iast_instrument_starlette_request_body(wrapped, instance, args, kwargs): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import origin_to_str - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - result = await wrapped(*args, **kwargs) return taint_pyobject( @@ -94,9 +86,6 @@ async def _iast_instrument_starlette_request_body(wrapped, instance, args, kwarg def _iast_instrument_starlette_scope(scope): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject - if scope.get("path_params"): try: for k, v in scope["path_params"].items(): diff --git a/ddtrace/appsec/_iast/_patches/json_tainting.py b/ddtrace/appsec/_iast/_patches/json_tainting.py index 28cfe41e592..44df9847ba1 100644 --- a/ddtrace/appsec/_iast/_patches/json_tainting.py +++ b/ddtrace/appsec/_iast/_patches/json_tainting.py @@ -43,8 +43,8 @@ def wrapped_loads(wrapped, instance, args, kwargs): obj = wrapped(*args, **kwargs) if asm_config._iast_enabled and is_iast_request_enabled(): - from .._taint_tracking import get_tainted_ranges - from .._taint_tracking import taint_pyobject + from .._taint_tracking._taint_objects import get_tainted_ranges + from .._taint_tracking._taint_objects import taint_pyobject ranges = get_tainted_ranges(args[0]) diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp index b7454de26f8..a08f76d9f3d 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp @@ -2,7 +2,7 @@ #include "Helpers.h" static PyObject* -do_modulo(PyObject* text, PyObject* insert_tuple_or_obj, py::object py_candidate_text, py::object py_candidate_tuple) +do_modulo(PyObject* text, PyObject* insert_tuple_or_obj) { PyObject* result = nullptr; @@ -13,22 +13,18 @@ do_modulo(PyObject* text, PyObject* insert_tuple_or_obj, py::object py_candidate Py_INCREF(insert_tuple); } else { insert_tuple = PyTuple_Pack(1, insert_tuple_or_obj); + if (insert_tuple == nullptr) { + return nullptr; + } } - if (PyUnicode_Check(text) && insert_tuple != nullptr) { + if (PyUnicode_Check(text)) { result = PyUnicode_Format(text, insert_tuple); + } else if (PyBytes_Check(text) or PyByteArray_Check(text)) { + auto method_name = PyUnicode_FromString("__mod__"); + result = PyObject_CallMethodObjArgs(text, method_name, insert_tuple, nullptr); + Py_DECREF(method_name); } else { - try { - py::object res_py = py_candidate_text.attr("__mod__")(py_candidate_tuple); - PyObject* res_pyo = res_py.ptr(); - if (res_pyo != nullptr) { - Py_INCREF(res_pyo); - } - return res_pyo; - } catch (py::error_already_set& e) { - e.restore(); - return nullptr; - } } Py_DECREF(insert_tuple); if (has_pyerr()) { @@ -53,7 +49,21 @@ api_modulo_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) // Lambda to get the result of the modulo operation auto get_result = [&]() -> PyObject* { - return do_modulo(candidate_text, candidate_tuple, py_candidate_text, py_candidate_tuple); + PyObject* res = do_modulo(candidate_text, candidate_tuple); + if (res == nullptr) { + try { + py::object res_py = py_candidate_text.attr("__mod__")(py_candidate_tuple); + PyObject* res_pyo = res_py.ptr(); + if (res_pyo != nullptr) { + Py_INCREF(res_pyo); + } + return res_pyo; + } catch (py::error_already_set& e) { + e.restore(); + return nullptr; + } + } + return res; }; TRY_CATCH_ASPECT("modulo_aspect", return get_result(), , { @@ -97,10 +107,7 @@ api_modulo_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) } py::tuple formatted_parameters(list_formatted_parameters); - PyObject* applied_params = do_modulo(StringToPyObject(fmttext, py_str_type).ptr(), - formatted_parameters.ptr(), - StringToPyObject(fmttext, py_str_type), - formatted_parameters); + PyObject* applied_params = do_modulo(StringToPyObject(fmttext, py_str_type).ptr(), formatted_parameters.ptr()); if (applied_params == nullptr) { return get_result(); } diff --git a/ddtrace/appsec/_iast/_taint_tracking/__init__.py b/ddtrace/appsec/_iast/_taint_tracking/__init__.py index 839f4b3537f..3dccbd2f345 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/__init__.py +++ b/ddtrace/appsec/_iast/_taint_tracking/__init__.py @@ -1,77 +1,49 @@ -from io import BytesIO -from io import StringIO -import itertools -from typing import TYPE_CHECKING # noqa:F401 -from typing import Any -from typing import Tuple - - -if TYPE_CHECKING: # pragma: no cover - from typing import Sequence # noqa:F401 - -from ddtrace.internal._unpatched import _threading as threading +from ddtrace.appsec._iast._taint_tracking._native import ops # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_format import _format_aspect # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_helpers import _convert_escaped_text_to_tainted_text + +# noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_helpers import as_formatted_evidence # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_helpers import common_replace # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_helpers import parse_params # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_helpers import set_ranges_on_splitted # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_split import _aspect_rsplit # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_split import _aspect_split # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspect_split import _aspect_splitlines # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathbasename # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathdirname # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathjoin # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathnormcase # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathsplit # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathsplitdrive # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathsplitext # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.aspects_ospath import _aspect_ospathsplitroot # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.initializer import active_map_addreses_size # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.initializer import debug_taint_map # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.initializer import initializer_size # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.initializer import num_objects_tainted # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import OriginType # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import Source # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import TagMappingMode # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import are_all_text_all_ranges # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import copy_and_shift_ranges_from_strings # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import copy_ranges_from_strings # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import get_range_by_hash # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import get_ranges # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import is_tainted # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import origin_to_str # noqa: F401 + +# noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import set_ranges # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import shift_taint_range # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import shift_taint_ranges # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import str_to_origin # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import taint_range as TaintRange # noqa: F401 from ddtrace.internal.logger import get_logger -from ..._constants import IAST -from ..._constants import IAST_SPAN_TAGS -from .._iast_request_context import is_iast_request_enabled -from .._metrics import _set_iast_error_metric -from .._metrics import _set_metric_iast_executed_source -from .._metrics import increment_iast_span_metric -from .._utils import _is_iast_debug_enabled -from .._utils import _is_iast_propagation_debug_enabled -from .._utils import _is_python_version_supported - log = get_logger(__name__) -if _is_python_version_supported(): - from ._native import ops - from ._native.aspect_format import _format_aspect - from ._native.aspect_helpers import _convert_escaped_text_to_tainted_text - from ._native.aspect_helpers import as_formatted_evidence - from ._native.aspect_helpers import common_replace - from ._native.aspect_helpers import parse_params - from ._native.aspect_helpers import set_ranges_on_splitted - from ._native.aspect_split import _aspect_rsplit - from ._native.aspect_split import _aspect_split - from ._native.aspect_split import _aspect_splitlines - from ._native.aspects_ospath import _aspect_ospathbasename - from ._native.aspects_ospath import _aspect_ospathdirname - from ._native.aspects_ospath import _aspect_ospathjoin - from ._native.aspects_ospath import _aspect_ospathnormcase - from ._native.aspects_ospath import _aspect_ospathsplit - from ._native.aspects_ospath import _aspect_ospathsplitdrive - from ._native.aspects_ospath import _aspect_ospathsplitext - from ._native.aspects_ospath import _aspect_ospathsplitroot - from ._native.initializer import active_map_addreses_size - from ._native.initializer import create_context - from ._native.initializer import debug_taint_map - from ._native.initializer import initializer_size - from ._native.initializer import num_objects_tainted - from ._native.initializer import reset_context - from ._native.initializer import reset_contexts - from ._native.taint_tracking import OriginType - from ._native.taint_tracking import Source - from ._native.taint_tracking import TagMappingMode - from ._native.taint_tracking import are_all_text_all_ranges - from ._native.taint_tracking import copy_and_shift_ranges_from_strings - from ._native.taint_tracking import copy_ranges_from_strings - from ._native.taint_tracking import get_range_by_hash - from ._native.taint_tracking import get_ranges - from ._native.taint_tracking import is_notinterned_notfasttainted_unicode - from ._native.taint_tracking import is_tainted - from ._native.taint_tracking import origin_to_str - from ._native.taint_tracking import set_fast_tainted_if_notinterned_unicode - from ._native.taint_tracking import set_ranges - from ._native.taint_tracking import shift_taint_range - from ._native.taint_tracking import shift_taint_ranges - from ._native.taint_tracking import str_to_origin - from ._native.taint_tracking import taint_range as TaintRange - - new_pyobject_id = ops.new_pyobject_id - set_ranges_from_values = ops.set_ranges_from_values - __all__ = [ "OriginType", "Source", @@ -103,10 +75,9 @@ "debug_taint_map", "get_range_by_hash", "get_ranges", - "iast_taint_log_error", "initializer_size", + "is_tainted", "is_notinterned_notfasttainted_unicode", - "is_pyobject_tainted", "modulo_aspect", "new_pyobject_id", "num_objects_tainted", @@ -121,198 +92,6 @@ "shift_taint_range", "shift_taint_ranges", "str_to_origin", - "taint_pyobject", ] - - -def iast_taint_log_error(msg): - if _is_iast_debug_enabled(): - import inspect - - stack = inspect.stack() - frame_info = "\n".join("%s %s" % (frame_info.filename, frame_info.lineno) for frame_info in stack[:7]) - log.debug("[IAST] Propagation error. %s:\n%s", msg, frame_info) - _set_iast_error_metric("[IAST] Propagation error. %s" % msg) - - -def is_pyobject_tainted(pyobject: Any) -> bool: - if not is_iast_request_enabled(): - return False - if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] - return False - - try: - return is_tainted(pyobject) - except ValueError as e: - iast_taint_log_error("Checking tainted object error: %s" % e) - return False - - -def _taint_pyobject_base(pyobject: Any, source_name: Any, source_value: Any, source_origin=None) -> Any: - if not is_iast_request_enabled(): - return pyobject - - if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] - return pyobject - # We need this validation in different condition if pyobject is not a text type and creates a side-effect such as - # __len__ magic method call. - pyobject_len = 0 - if isinstance(pyobject, IAST.TEXT_TYPES): - pyobject_len = len(pyobject) - if pyobject_len == 0: - return pyobject - - if isinstance(source_name, (bytes, bytearray)): - source_name = str(source_name, encoding="utf8", errors="ignore") - if isinstance(source_name, OriginType): - source_name = origin_to_str(source_name) - - if isinstance(source_value, (bytes, bytearray)): - source_value = str(source_value, encoding="utf8", errors="ignore") - if source_origin is None: - source_origin = OriginType.PARAMETER - - try: - pyobject_newid = set_ranges_from_values(pyobject, pyobject_len, source_name, source_value, source_origin) - return pyobject_newid - except ValueError as e: - log.debug("Tainting object error (pyobject type %s): %s", type(pyobject), e, exc_info=True) - return pyobject - - -def taint_pyobject(pyobject: Any, source_name: Any, source_value: Any, source_origin=None) -> Any: - try: - if source_origin is None: - source_origin = OriginType.PARAMETER - - res = _taint_pyobject_base(pyobject, source_name, source_value, source_origin) - _set_metric_iast_executed_source(source_origin) - increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SOURCE, source_origin) - return res - except ValueError as e: - log.debug("Tainting object error (pyobject type %s): %s", type(pyobject), e) - return pyobject - - -def taint_pyobject_with_ranges(pyobject: Any, ranges: Tuple) -> bool: - if not is_iast_request_enabled(): - return False - if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] - return False - try: - set_ranges(pyobject, ranges) - return True - except ValueError as e: - iast_taint_log_error("Tainting object with ranges error (pyobject type %s): %s" % (type(pyobject), e)) - return False - - -def get_tainted_ranges(pyobject: Any) -> Tuple: - if not is_iast_request_enabled(): - return tuple() - if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] - return tuple() - try: - return get_ranges(pyobject) - except ValueError as e: - iast_taint_log_error("Get ranges error (pyobject type %s): %s" % (type(pyobject), e)) - return tuple() - - -if _is_iast_propagation_debug_enabled(): - TAINTED_FRAMES = [] - - def trace_calls_and_returns(frame, event, arg): - co = frame.f_code - func_name = co.co_name - if func_name == "write": - # Ignore write() calls from print statements - return - if func_name in ("is_pyobject_tainted", "__repr__"): - return - line_no = frame.f_lineno - filename = co.co_filename - if "ddtrace" in filename: - return - if event == "call": - f_locals = frame.f_locals - try: - if any([is_pyobject_tainted(f_locals[arg]) for arg in f_locals]): - TAINTED_FRAMES.append(frame) - log.debug("Call to %s on line %s of %s, args: %s", func_name, line_no, filename, frame.f_locals) - log.debug("Tainted arguments:") - for arg in f_locals: - if is_pyobject_tainted(f_locals[arg]): - log.debug("\t%s: %s", arg, f_locals[arg]) - log.debug("-----") - return trace_calls_and_returns - except AttributeError: - pass - elif event == "return": - if frame in TAINTED_FRAMES: - TAINTED_FRAMES.remove(frame) - log.debug("Return from %s on line %d of %s, return value: %s", func_name, line_no, filename, arg) - if isinstance(arg, (str, bytes, bytearray, BytesIO, StringIO, list, tuple, dict)): - if ( - (isinstance(arg, (str, bytes, bytearray, BytesIO, StringIO)) and is_pyobject_tainted(arg)) - or (isinstance(arg, (list, tuple)) and any([is_pyobject_tainted(x) for x in arg])) - or (isinstance(arg, dict) and any([is_pyobject_tainted(x) for x in arg.values()])) - ): - log.debug("Return value is tainted") - else: - log.debug("Return value is NOT tainted") - log.debug("-----") - return - - threading.settrace(trace_calls_and_returns) - - -def copy_ranges_to_string(pyobject, ranges): - # type: (str, Sequence[TaintRange]) -> str - # NB this function uses comment-based type annotation because TaintRange is conditionally imported - if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] - return pyobject - - for r in ranges: - _is_string_in_source_value = False - if r.source.value: - if isinstance(pyobject, (bytes, bytearray)): - pyobject_str = str(pyobject, encoding="utf8", errors="ignore") - else: - pyobject_str = pyobject - _is_string_in_source_value = pyobject_str in r.source.value - - if _is_string_in_source_value: - pyobject = _taint_pyobject_base( - pyobject=pyobject, - source_name=r.source.name, - source_value=r.source.value, - source_origin=r.source.origin, - ) - break - else: - # no total match found, maybe partial match, just take the first one - pyobject = _taint_pyobject_base( - pyobject=pyobject, - source_name=ranges[0].source.name, - source_value=ranges[0].source.value, - source_origin=ranges[0].source.origin, - ) - return pyobject - - -# Given a list of ranges, try to match them with the iterable and return a new iterable with a new range applied that -# matched the original one Source. If no range matches, take the Source from the first one. -def copy_ranges_to_iterable_with_strings(iterable, ranges): - # type: (Sequence[str], Sequence[TaintRange]) -> Sequence[str] - # NB this function uses comment-based type annotation because TaintRange is conditionally imported - iterable_type = type(iterable) - - new_result = [] - # do this so it doesn't consume a potential generator - items, items_backup = itertools.tee(iterable) - for i in items_backup: - i = copy_ranges_to_string(i, ranges) - new_result.append(i) - - return iterable_type(new_result) # type: ignore[call-arg] +new_pyobject_id = ops.new_pyobject_id +set_ranges_from_values = ops.set_ranges_from_values diff --git a/ddtrace/appsec/_iast/_taint_tracking/_context.py b/ddtrace/appsec/_iast/_taint_tracking/_context.py new file mode 100644 index 00000000000..160d229faec --- /dev/null +++ b/ddtrace/appsec/_iast/_taint_tracking/_context.py @@ -0,0 +1,10 @@ +from ddtrace.appsec._iast._taint_tracking._native.initializer import create_context # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.initializer import reset_context # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._native.initializer import reset_contexts # noqa: F401 + + +__all__ = [ + "create_context", + "reset_context", + "reset_contexts", +] diff --git a/ddtrace/appsec/_iast/_taint_tracking/_debug.py b/ddtrace/appsec/_iast/_taint_tracking/_debug.py new file mode 100644 index 00000000000..6b7e6ec4d3d --- /dev/null +++ b/ddtrace/appsec/_iast/_taint_tracking/_debug.py @@ -0,0 +1,57 @@ +from io import BytesIO +from io import StringIO + +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._utils import _is_iast_propagation_debug_enabled +from ddtrace.internal._unpatched import _threading as threading +from ddtrace.internal.logger import get_logger + + +log = get_logger(__name__) + +if _is_iast_propagation_debug_enabled(): + TAINTED_FRAMES = [] + + def trace_calls_and_returns(frame, event, arg): + co = frame.f_code + func_name = co.co_name + if func_name == "write": + # Ignore write() calls from print statements + return + if func_name in ("is_pyobject_tainted", "__repr__"): + return + line_no = frame.f_lineno + filename = co.co_filename + if "ddtrace" in filename: + return + if event == "call": + f_locals = frame.f_locals + try: + if any([is_pyobject_tainted(f_locals[arg]) for arg in f_locals]): + TAINTED_FRAMES.append(frame) + log.debug("Call to %s on line %s of %s, args: %s", func_name, line_no, filename, frame.f_locals) + log.debug("Tainted arguments:") + for arg in f_locals: + if is_pyobject_tainted(f_locals[arg]): + log.debug("\t%s: %s", arg, f_locals[arg]) + log.debug("-----") + return trace_calls_and_returns + except AttributeError: + pass + elif event == "return": + if frame in TAINTED_FRAMES: + TAINTED_FRAMES.remove(frame) + log.debug("Return from %s on line %d of %s, return value: %s", func_name, line_no, filename, arg) + if isinstance(arg, (str, bytes, bytearray, BytesIO, StringIO, list, tuple, dict)): + if ( + (isinstance(arg, (str, bytes, bytearray, BytesIO, StringIO)) and is_pyobject_tainted(arg)) + or (isinstance(arg, (list, tuple)) and any([is_pyobject_tainted(x) for x in arg])) + or (isinstance(arg, dict) and any([is_pyobject_tainted(x) for x in arg.values()])) + ): + log.debug("Return value is tainted") + else: + log.debug("Return value is NOT tainted") + log.debug("-----") + return + + threading.settrace(trace_calls_and_returns) diff --git a/ddtrace/appsec/_iast/_taint_tracking/_errors.py b/ddtrace/appsec/_iast/_taint_tracking/_errors.py new file mode 100644 index 00000000000..0d7c2fb856b --- /dev/null +++ b/ddtrace/appsec/_iast/_taint_tracking/_errors.py @@ -0,0 +1,16 @@ +import inspect + +from ddtrace.appsec._iast._metrics import _set_iast_error_metric +from ddtrace.appsec._iast._utils import _is_iast_debug_enabled +from ddtrace.internal.logger import get_logger + + +log = get_logger(__name__) + + +def iast_taint_log_error(msg): + if _is_iast_debug_enabled(): + stack = inspect.stack() + frame_info = "\n".join("%s %s" % (frame_info.filename, frame_info.lineno) for frame_info in stack[:7]) + log.debug("[IAST] Propagation error. %s:\n%s", msg, frame_info) + _set_iast_error_metric("[IAST] Propagation error. %s" % msg) diff --git a/ddtrace/appsec/_iast/_taint_tracking/_native.cpp b/ddtrace/appsec/_iast/_taint_tracking/_native.cpp index 170c12d8429..e0605a853b5 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/_native.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/_native.cpp @@ -64,19 +64,6 @@ static struct PyModuleDef ops = { PyModuleDef_HEAD_INIT, */ PYBIND11_MODULE(_native, m) { - const char* env_iast_enabled = std::getenv("DD_IAST_ENABLED"); - if (env_iast_enabled == nullptr) { - py::module::import("logging").attr("warning")("IAST not enabled but native module is being loaded"); - } else { - std::string iast_enabled = std::string(env_iast_enabled); - std::transform(iast_enabled.begin(), iast_enabled.end(), iast_enabled.begin(), [](unsigned char c) { - return std::tolower(c); - }); - if (iast_enabled != "true" && iast_enabled != "1") { - py::module::import("logging").attr("warning")("IAST not enabled but native module is being loaded"); - } - } - initializer = make_unique(); // Create a atexit callback to cleanup the Initializer before the interpreter finishes diff --git a/ddtrace/appsec/_iast/_taint_tracking/_taint_objects.py b/ddtrace/appsec/_iast/_taint_tracking/_taint_objects.py new file mode 100644 index 00000000000..660e8b8e69b --- /dev/null +++ b/ddtrace/appsec/_iast/_taint_tracking/_taint_objects.py @@ -0,0 +1,154 @@ +import itertools +from typing import Any +from typing import Sequence +from typing import Tuple + +from ddtrace.appsec._constants import IAST +from ddtrace.appsec._constants import IAST_SPAN_TAGS +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_source +from ddtrace.appsec._iast._metrics import increment_iast_span_metric +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking import TaintRange +from ddtrace.appsec._iast._taint_tracking import get_ranges +from ddtrace.appsec._iast._taint_tracking import is_tainted +from ddtrace.appsec._iast._taint_tracking import origin_to_str +from ddtrace.appsec._iast._taint_tracking import set_ranges +from ddtrace.appsec._iast._taint_tracking import set_ranges_from_values +from ddtrace.appsec._iast._taint_tracking._errors import iast_taint_log_error +from ddtrace.internal.logger import get_logger + + +log = get_logger(__name__) + + +def is_pyobject_tainted(pyobject: Any) -> bool: + if not is_iast_request_enabled(): + return False + if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] + return False + + try: + return is_tainted(pyobject) + except ValueError as e: + iast_taint_log_error("Checking tainted object error: %s" % e) + return False + + +def _taint_pyobject_base(pyobject: Any, source_name: Any, source_value: Any, source_origin=None) -> Any: + if not is_iast_request_enabled(): + return pyobject + + if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] + return pyobject + # We need this validation in different condition if pyobject is not a text type and creates a side-effect such as + # __len__ magic method call. + pyobject_len = 0 + if isinstance(pyobject, IAST.TEXT_TYPES): + pyobject_len = len(pyobject) + if pyobject_len == 0: + return pyobject + + if isinstance(source_name, (bytes, bytearray)): + source_name = str(source_name, encoding="utf8", errors="ignore") + if isinstance(source_name, OriginType): + source_name = origin_to_str(source_name) + + if isinstance(source_value, (bytes, bytearray)): + source_value = str(source_value, encoding="utf8", errors="ignore") + if source_origin is None: + source_origin = OriginType.PARAMETER + + try: + pyobject_newid = set_ranges_from_values(pyobject, pyobject_len, source_name, source_value, source_origin) + return pyobject_newid + except ValueError as e: + log.debug("Tainting object error (pyobject type %s): %s", type(pyobject), e, exc_info=True) + return pyobject + + +def taint_pyobject_with_ranges(pyobject: Any, ranges: Tuple) -> bool: + if not is_iast_request_enabled(): + return False + if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] + return False + try: + set_ranges(pyobject, ranges) + return True + except ValueError as e: + iast_taint_log_error("Tainting object with ranges error (pyobject type %s): %s" % (type(pyobject), e)) + return False + + +def get_tainted_ranges(pyobject: Any) -> Tuple: + if not is_iast_request_enabled(): + return tuple() + if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] + return tuple() + try: + return get_ranges(pyobject) + except ValueError as e: + iast_taint_log_error("Get ranges error (pyobject type %s): %s" % (type(pyobject), e)) + return tuple() + + +def taint_pyobject(pyobject: Any, source_name: Any, source_value: Any, source_origin=None) -> Any: + try: + if source_origin is None: + source_origin = OriginType.PARAMETER + + res = _taint_pyobject_base(pyobject, source_name, source_value, source_origin) + _set_metric_iast_executed_source(source_origin) + increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SOURCE, source_origin) + return res + except ValueError as e: + log.debug("Tainting object error (pyobject type %s): %s", type(pyobject), e) + return pyobject + + +def copy_ranges_to_string(pyobject: str, ranges: Sequence[TaintRange]) -> str: + # NB this function uses comment-based type annotation because TaintRange is conditionally imported + if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] + return pyobject + + for r in ranges: + _is_string_in_source_value = False + if r.source.value: + if isinstance(pyobject, (bytes, bytearray)): + pyobject_str = str(pyobject, encoding="utf8", errors="ignore") + else: + pyobject_str = pyobject + _is_string_in_source_value = pyobject_str in r.source.value + + if _is_string_in_source_value: + pyobject = _taint_pyobject_base( + pyobject=pyobject, + source_name=r.source.name, + source_value=r.source.value, + source_origin=r.source.origin, + ) + break + else: + # no total match found, maybe partial match, just take the first one + pyobject = _taint_pyobject_base( + pyobject=pyobject, + source_name=ranges[0].source.name, + source_value=ranges[0].source.value, + source_origin=ranges[0].source.origin, + ) + return pyobject + + +def copy_ranges_to_iterable_with_strings(iterable, ranges): + # type: (Sequence[str], Sequence[TaintRange]) -> Sequence[str] + # NB this function uses comment-based type annotation because TaintRange is conditionally imported + iterable_type = type(iterable) + + new_result = [] + # do this so it doesn't consume a potential generator + items, items_backup = itertools.tee(iterable) + for i in items_backup: + i = copy_ranges_to_string(i, ranges) + new_result.append(i) + + return iterable_type(new_result) # type: ignore[call-arg] diff --git a/ddtrace/appsec/_iast/_taint_tracking/aspects.py b/ddtrace/appsec/_iast/_taint_tracking/aspects.py index d70dc76449c..925cd4a5b9d 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/aspects.py +++ b/ddtrace/appsec/_iast/_taint_tracking/aspects.py @@ -19,39 +19,38 @@ import _io from ddtrace.appsec._constants import IAST - -from .._taint_tracking import TagMappingMode -from .._taint_tracking import TaintRange -from .._taint_tracking import _aspect_ospathbasename -from .._taint_tracking import _aspect_ospathdirname -from .._taint_tracking import _aspect_ospathjoin -from .._taint_tracking import _aspect_ospathnormcase -from .._taint_tracking import _aspect_ospathsplit -from .._taint_tracking import _aspect_ospathsplitdrive -from .._taint_tracking import _aspect_ospathsplitext -from .._taint_tracking import _aspect_ospathsplitroot -from .._taint_tracking import _aspect_rsplit -from .._taint_tracking import _aspect_split -from .._taint_tracking import _aspect_splitlines -from .._taint_tracking import _convert_escaped_text_to_tainted_text -from .._taint_tracking import _format_aspect -from .._taint_tracking import are_all_text_all_ranges -from .._taint_tracking import as_formatted_evidence -from .._taint_tracking import common_replace -from .._taint_tracking import copy_and_shift_ranges_from_strings -from .._taint_tracking import copy_ranges_from_strings -from .._taint_tracking import copy_ranges_to_iterable_with_strings -from .._taint_tracking import copy_ranges_to_string -from .._taint_tracking import get_ranges -from .._taint_tracking import get_tainted_ranges -from .._taint_tracking import iast_taint_log_error -from .._taint_tracking import is_pyobject_tainted -from .._taint_tracking import new_pyobject_id -from .._taint_tracking import parse_params -from .._taint_tracking import set_ranges -from .._taint_tracking import shift_taint_range -from .._taint_tracking import taint_pyobject_with_ranges -from .._taint_tracking._native import aspects # noqa: F401 +from ddtrace.appsec._iast._taint_tracking import TagMappingMode +from ddtrace.appsec._iast._taint_tracking import TaintRange +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathbasename +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathdirname +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathjoin +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathnormcase +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathsplit +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathsplitdrive +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathsplitext +from ddtrace.appsec._iast._taint_tracking import _aspect_ospathsplitroot +from ddtrace.appsec._iast._taint_tracking import _aspect_rsplit +from ddtrace.appsec._iast._taint_tracking import _aspect_split +from ddtrace.appsec._iast._taint_tracking import _aspect_splitlines +from ddtrace.appsec._iast._taint_tracking import _convert_escaped_text_to_tainted_text +from ddtrace.appsec._iast._taint_tracking import _format_aspect +from ddtrace.appsec._iast._taint_tracking import are_all_text_all_ranges +from ddtrace.appsec._iast._taint_tracking import as_formatted_evidence +from ddtrace.appsec._iast._taint_tracking import common_replace +from ddtrace.appsec._iast._taint_tracking import copy_and_shift_ranges_from_strings +from ddtrace.appsec._iast._taint_tracking import copy_ranges_from_strings +from ddtrace.appsec._iast._taint_tracking import get_ranges +from ddtrace.appsec._iast._taint_tracking import new_pyobject_id +from ddtrace.appsec._iast._taint_tracking import parse_params +from ddtrace.appsec._iast._taint_tracking import set_ranges +from ddtrace.appsec._iast._taint_tracking import shift_taint_range +from ddtrace.appsec._iast._taint_tracking._errors import iast_taint_log_error +from ddtrace.appsec._iast._taint_tracking._native import aspects # noqa: F401 +from ddtrace.appsec._iast._taint_tracking._taint_objects import copy_ranges_to_iterable_with_strings +from ddtrace.appsec._iast._taint_tracking._taint_objects import copy_ranges_to_string +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject_with_ranges TEXT_TYPES = Union[str, bytes, bytearray] diff --git a/ddtrace/appsec/_iast/_taint_utils.py b/ddtrace/appsec/_iast/_taint_utils.py index 8b5e1b97caa..524e8279d2b 100644 --- a/ddtrace/appsec/_iast/_taint_utils.py +++ b/ddtrace/appsec/_iast/_taint_utils.py @@ -5,6 +5,8 @@ from typing import Optional from typing import Union +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import DBAPI_INTEGRATIONS from ddtrace.internal.logger import get_logger from ddtrace.settings.asm import config as asm_config @@ -87,9 +89,6 @@ def taint_structure(main_obj, source_key, source_value, override_pyobject_tainte use a queue like mechanism to avoid recursion Best effort: mutate mutable structures and rebuild immutable ones if possible """ - from ._taint_tracking import is_pyobject_tainted - from ._taint_tracking import taint_pyobject - if not main_obj: return main_obj @@ -164,9 +163,6 @@ def __init__(self, original_list, origins=(0, 0), override_pyobject_tainted=Fals def _taint(self, value): if value: if isinstance(value, (str, bytes, bytearray)): - from ._taint_tracking import is_pyobject_tainted - from ._taint_tracking import taint_pyobject - if not is_pyobject_tainted(value) or self._override_pyobject_tainted: try: # TODO: migrate this part to shift ranges instead of creating a new one @@ -348,9 +344,6 @@ def _taint(self, value, key, origin=None): origin = self._origin_value if value: if isinstance(value, (str, bytes, bytearray)): - from ._taint_tracking import is_pyobject_tainted - from ._taint_tracking import taint_pyobject - if not is_pyobject_tainted(value) or self._override_pyobject_tainted: try: # TODO: migrate this part to shift ranges instead of creating a new one @@ -529,8 +522,6 @@ def supported_dbapi_integration(integration_name): def check_tainted_dbapi_args(args, kwargs, tracer, integration_name, method): if supported_dbapi_integration(integration_name) and method.__name__ == "execute": - from ._taint_tracking import is_pyobject_tainted - return len(args) and args[0] and is_pyobject_tainted(args[0]) return False diff --git a/ddtrace/appsec/_iast/_utils.py b/ddtrace/appsec/_iast/_utils.py index c1ae2d82be4..fda05a8b8e5 100644 --- a/ddtrace/appsec/_iast/_utils.py +++ b/ddtrace/appsec/_iast/_utils.py @@ -8,8 +8,8 @@ @lru_cache(maxsize=1) def _is_python_version_supported() -> bool: - # IAST supports Python versions 3.6 to 3.12 - return (3, 6, 0) <= sys.version_info < (3, 13, 0) + # IAST supports Python versions 3.6 to 3.13 + return (3, 6, 0) <= sys.version_info < (3, 14, 0) def _is_iast_enabled(): diff --git a/ddtrace/appsec/_iast/reporter.py b/ddtrace/appsec/_iast/reporter.py index 62cc2ee8d65..ffdd786fc28 100644 --- a/ddtrace/appsec/_iast/reporter.py +++ b/ddtrace/appsec/_iast/reporter.py @@ -218,7 +218,7 @@ def taint_ranges_as_evidence_info(pyobject: Any) -> Tuple[List[Source], List[Dic Returns: - Tuple[Set[Source], List[Dict]]: Set of Source objects and list of tainted ranges as dictionaries. """ - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges sources = list() tainted_ranges = get_tainted_ranges(pyobject) diff --git a/ddtrace/appsec/_iast/taint_sinks/command_injection.py b/ddtrace/appsec/_iast/taint_sinks/command_injection.py index 0cfd48a5816..ee22b294bfc 100644 --- a/ddtrace/appsec/_iast/taint_sinks/command_injection.py +++ b/ddtrace/appsec/_iast/taint_sinks/command_injection.py @@ -3,17 +3,19 @@ from typing import List from typing import Union +from ddtrace.appsec._common_module_patches import try_unwrap +from ddtrace.appsec._constants import IAST_SPAN_TAGS +from ddtrace.appsec._iast import oce +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_sink +from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink +from ddtrace.appsec._iast._metrics import increment_iast_span_metric +from ddtrace.appsec._iast._patch import try_wrap_function_wrapper +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast.constants import VULN_CMDI from ddtrace.internal.logger import get_logger from ddtrace.settings.asm import config as asm_config -from ..._common_module_patches import try_unwrap -from ..._constants import IAST_SPAN_TAGS -from .. import oce -from .._iast_request_context import is_iast_request_enabled -from .._metrics import _set_metric_iast_instrumented_sink -from .._metrics import increment_iast_span_metric -from .._patch import try_wrap_function_wrapper -from ..constants import VULN_CMDI from ._base import VulnerabilityBase @@ -75,13 +77,11 @@ class CommandInjection(VulnerabilityBase): def _iast_report_cmdi(shell_args: Union[str, List[str]]) -> None: report_cmdi = "" - from .._metrics import _set_metric_iast_executed_sink increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, CommandInjection.vulnerability_type) _set_metric_iast_executed_sink(CommandInjection.vulnerability_type) if is_iast_request_enabled() and CommandInjection.has_quota(): - from .._taint_tracking import is_pyobject_tainted from .._taint_tracking.aspects import join_aspect if isinstance(shell_args, (list, tuple)): diff --git a/ddtrace/appsec/_iast/taint_sinks/header_injection.py b/ddtrace/appsec/_iast/taint_sinks/header_injection.py index 4d56986c2d0..730e9f05490 100644 --- a/ddtrace/appsec/_iast/taint_sinks/header_injection.py +++ b/ddtrace/appsec/_iast/taint_sinks/header_injection.py @@ -2,20 +2,22 @@ from wrapt.importer import when_imported +from ddtrace.appsec._common_module_patches import try_unwrap +from ddtrace.appsec._constants import IAST_SPAN_TAGS +from ddtrace.appsec._iast import oce +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_sink +from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink +from ddtrace.appsec._iast._metrics import increment_iast_span_metric +from ddtrace.appsec._iast._patch import set_and_check_module_is_patched +from ddtrace.appsec._iast._patch import set_module_unpatched +from ddtrace.appsec._iast._patch import try_wrap_function_wrapper +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast.constants import HEADER_NAME_VALUE_SEPARATOR +from ddtrace.appsec._iast.constants import VULN_HEADER_INJECTION from ddtrace.internal.logger import get_logger from ddtrace.settings.asm import config as asm_config -from ..._common_module_patches import try_unwrap -from ..._constants import IAST_SPAN_TAGS -from .. import oce -from .._iast_request_context import is_iast_request_enabled -from .._metrics import _set_metric_iast_instrumented_sink -from .._metrics import increment_iast_span_metric -from .._patch import set_and_check_module_is_patched -from .._patch import set_module_unpatched -from .._patch import try_wrap_function_wrapper -from ..constants import HEADER_NAME_VALUE_SEPARATOR -from ..constants import VULN_HEADER_INJECTION from ._base import VulnerabilityBase @@ -97,9 +99,7 @@ class HeaderInjection(VulnerabilityBase): def _iast_report_header_injection(headers_args) -> None: - from .._metrics import _set_metric_iast_executed_sink - from .._taint_tracking import is_pyobject_tainted - from .._taint_tracking.aspects import add_aspect + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect header_name, header_value = headers_args for header_to_exclude in HEADER_INJECTION_EXCLUSIONS: diff --git a/ddtrace/appsec/_iast/taint_sinks/insecure_cookie.py b/ddtrace/appsec/_iast/taint_sinks/insecure_cookie.py index f4cb00fc433..3e9c05c451c 100644 --- a/ddtrace/appsec/_iast/taint_sinks/insecure_cookie.py +++ b/ddtrace/appsec/_iast/taint_sinks/insecure_cookie.py @@ -7,7 +7,7 @@ from .._iast_request_context import is_iast_request_enabled from .._metrics import _set_metric_iast_executed_sink from .._metrics import increment_iast_span_metric -from .._taint_tracking import iast_taint_log_error +from .._taint_tracking._errors import iast_taint_log_error from ..constants import VULN_INSECURE_COOKIE from ..constants import VULN_NO_HTTPONLY_COOKIE from ..constants import VULN_NO_SAMESITE_COOKIE diff --git a/ddtrace/appsec/_iast/taint_sinks/path_traversal.py b/ddtrace/appsec/_iast/taint_sinks/path_traversal.py index 1fd9cff8956..42c1045c7ce 100644 --- a/ddtrace/appsec/_iast/taint_sinks/path_traversal.py +++ b/ddtrace/appsec/_iast/taint_sinks/path_traversal.py @@ -1,12 +1,14 @@ from typing import Any +from ddtrace.appsec._constants import IAST_SPAN_TAGS +from ddtrace.appsec._iast import oce +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_sink +from ddtrace.appsec._iast._metrics import increment_iast_span_metric +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL from ddtrace.internal.logger import get_logger -from ..._constants import IAST_SPAN_TAGS -from .. import oce -from .._iast_request_context import is_iast_request_enabled -from .._metrics import increment_iast_span_metric -from ..constants import VULN_PATH_TRAVERSAL from ._base import VulnerabilityBase @@ -21,9 +23,6 @@ class PathTraversal(VulnerabilityBase): def check_and_report_path_traversal(*args: Any, **kwargs: Any) -> None: if is_iast_request_enabled() and PathTraversal.has_quota(): try: - from .._metrics import _set_metric_iast_executed_sink - from .._taint_tracking import is_pyobject_tainted - increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, PathTraversal.vulnerability_type) _set_metric_iast_executed_sink(PathTraversal.vulnerability_type) filename_arg = args[0] if args else kwargs.get("file", None) diff --git a/ddtrace/appsec/_iast/taint_sinks/ssrf.py b/ddtrace/appsec/_iast/taint_sinks/ssrf.py index 7233aa54cec..5090e73bc76 100644 --- a/ddtrace/appsec/_iast/taint_sinks/ssrf.py +++ b/ddtrace/appsec/_iast/taint_sinks/ssrf.py @@ -1,15 +1,17 @@ from typing import Callable +from ddtrace.appsec._constants import IAST_SPAN_TAGS +from ddtrace.appsec._iast import oce +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_sink +from ddtrace.appsec._iast._metrics import increment_iast_span_metric +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.internal.logger import get_logger from ddtrace.internal.utils import ArgumentError from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.importlib import func_name -from ..._constants import IAST_SPAN_TAGS -from .. import oce -from .._iast_request_context import is_iast_request_enabled -from .._metrics import increment_iast_span_metric -from ..constants import VULN_SSRF from ._base import VulnerabilityBase @@ -46,14 +48,10 @@ def _iast_report_ssrf(func: Callable, *args, **kwargs): return if report_ssrf: - from .._metrics import _set_metric_iast_executed_sink - _set_metric_iast_executed_sink(SSRF.vulnerability_type) increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, SSRF.vulnerability_type) if is_iast_request_enabled() and SSRF.has_quota(): try: - from .._taint_tracking import is_pyobject_tainted - if is_pyobject_tainted(report_ssrf): SSRF.report(evidence_value=report_ssrf) except Exception: diff --git a/ddtrace/contrib/internal/asgi/middleware.py b/ddtrace/contrib/internal/asgi/middleware.py index e32d2994a3d..98d352cf75f 100644 --- a/ddtrace/contrib/internal/asgi/middleware.py +++ b/ddtrace/contrib/internal/asgi/middleware.py @@ -91,6 +91,18 @@ async def _blocked_asgi_app(scope, receive, send): await send({"type": "http.response.body", "body": b""}) +def _parse_response_cookies(response_headers): + cookies = {} + try: + result = response_headers.get("set-cookie", "").split("=", maxsplit=1) + if len(result) == 2: + cookie_key, cookie_value = result + cookies[cookie_key] = cookie_value + except Exception: + log.debug("failed to extract response cookies", exc_info=True) + return cookies + + class TraceMiddleware: """ ASGI application middleware that traces the requests. @@ -211,7 +223,6 @@ async def __call__(self, scope, receive, send): peer_ip = client[0] else: peer_ip = None - trace_utils.set_http_meta( span, self.integration_config, @@ -234,15 +245,8 @@ async def wrapped_send(message): except Exception: log.warning("failed to extract response headers", exc_info=True) response_headers = None - if span and message.get("type") == "http.response.start" and "status" in message: - cookies = {} - try: - cookie_key, cookie_value = response_headers.get("set-cookie", "").split("=", maxsplit=1) - cookies[cookie_key] = cookie_value - except Exception: - log.debug("failed to extract response cookies", exc_info=True) - + cookies = _parse_response_cookies(response_headers) status_code = message["status"] trace_utils.set_http_meta( span, diff --git a/ddtrace/contrib/internal/botocore/services/kinesis.py b/ddtrace/contrib/internal/botocore/services/kinesis.py index 1f8bbef1478..0287c29d2bc 100644 --- a/ddtrace/contrib/internal/botocore/services/kinesis.py +++ b/ddtrace/contrib/internal/botocore/services/kinesis.py @@ -1,5 +1,6 @@ from datetime import datetime import json +from time import time_ns from typing import Any from typing import Dict from typing import List @@ -12,7 +13,6 @@ from ddtrace.contrib.trace_utils import ext_service from ddtrace.ext import SpanTypes from ddtrace.internal import core -from ddtrace.internal.compat import time_ns from ddtrace.internal.logger import get_logger from ddtrace.internal.schema import schematize_cloud_messaging_operation from ddtrace.internal.schema import schematize_service_name diff --git a/ddtrace/contrib/internal/kafka/patch.py b/ddtrace/contrib/internal/kafka/patch.py index b8e8fce007d..339e2469914 100644 --- a/ddtrace/contrib/internal/kafka/patch.py +++ b/ddtrace/contrib/internal/kafka/patch.py @@ -1,5 +1,6 @@ import os import sys +from time import time_ns import confluent_kafka @@ -12,7 +13,6 @@ from ddtrace.ext import SpanTypes from ddtrace.ext import kafka as kafkax from ddtrace.internal import core -from ddtrace.internal.compat import time_ns from ddtrace.internal.constants import COMPONENT from ddtrace.internal.constants import MESSAGING_SYSTEM from ddtrace.internal.logger import get_logger diff --git a/ddtrace/contrib/internal/langchain/patch.py b/ddtrace/contrib/internal/langchain/patch.py index fa2332d70f2..b7513539da7 100644 --- a/ddtrace/contrib/internal/langchain/patch.py +++ b/ddtrace/contrib/internal/langchain/patch.py @@ -1406,8 +1406,8 @@ def unpatch(): def taint_outputs(instance, inputs, outputs): from ddtrace.appsec._iast._metrics import _set_iast_error_metric - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject try: ranges = None @@ -1429,8 +1429,8 @@ def taint_outputs(instance, inputs, outputs): def taint_parser_output(func, instance, args, kwargs): from ddtrace.appsec._iast._metrics import _set_iast_error_metric - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject result = func(*args, **kwargs) try: diff --git a/ddtrace/contrib/pytest/_atr_utils.py b/ddtrace/contrib/pytest/_atr_utils.py index 89be8b881af..0d684486602 100644 --- a/ddtrace/contrib/pytest/_atr_utils.py +++ b/ddtrace/contrib/pytest/_atr_utils.py @@ -29,33 +29,56 @@ class _ATR_RETRY_OUTCOMES: ATR_FINAL_FAILED = "dd_atr_final_failed" +class _QUARANTINE_ATR_RETRY_OUTCOMES(_ATR_RETRY_OUTCOMES): + ATR_ATTEMPT_PASSED = "dd_quarantine_atr_attempt_passed" + ATR_ATTEMPT_FAILED = "dd_quarantine_atr_attempt_failed" + ATR_ATTEMPT_SKIPPED = "dd_quarantine_atr_attempt_skipped" + ATR_FINAL_PASSED = "dd_quarantine_atr_final_passed" + ATR_FINAL_FAILED = "dd_quarantine_atr_final_failed" + + _FINAL_OUTCOMES: t.Dict[TestStatus, str] = { TestStatus.PASS: _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, TestStatus.FAIL: _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, } +_QUARANTINE_FINAL_OUTCOMES: t.Dict[TestStatus, str] = { + TestStatus.PASS: _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + TestStatus.FAIL: _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, +} + + def atr_handle_retries( test_id: InternalTestId, item: pytest.Item, when: str, original_result: pytest_TestReport, test_outcome: _TestOutcome, + is_quarantined: bool = False, ): + if is_quarantined: + retry_outcomes = _QUARANTINE_ATR_RETRY_OUTCOMES + final_outcomes = _QUARANTINE_FINAL_OUTCOMES + else: + retry_outcomes = _ATR_RETRY_OUTCOMES + final_outcomes = _FINAL_OUTCOMES + + outcomes = RetryOutcomes( + PASSED=retry_outcomes.ATR_ATTEMPT_PASSED, + FAILED=retry_outcomes.ATR_ATTEMPT_FAILED, + SKIPPED=retry_outcomes.ATR_ATTEMPT_SKIPPED, + XFAIL=retry_outcomes.ATR_ATTEMPT_PASSED, + XPASS=retry_outcomes.ATR_ATTEMPT_FAILED, + ) + # Overwrite the original result to avoid double-counting when displaying totals in final summary if when == "call": if test_outcome.status == TestStatus.FAIL: - original_result.outcome = _ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED - return - if InternalTest.get_tag(test_id, "_dd.ci.atr_setup_failed"): - log.debug("Test item %s failed during setup, will not be retried for Early Flake Detection") - return - if InternalTest.get_tag(test_id, "_dd.ci.atr_teardown_failed"): - # NOTE: tests that passed their call but failed during teardown are not retried - log.debug("Test item %s failed during teardown, will not be retried for Early Flake Detection") + original_result.outcome = outcomes.FAILED return - atr_outcome = _atr_do_retries(item) + atr_outcome = _atr_do_retries(item, outcomes) final_report = pytest_TestReport( nodeid=item.nodeid, @@ -63,7 +86,7 @@ def atr_handle_retries( keywords=item.keywords, when="call", longrepr=None, - outcome=_FINAL_OUTCOMES[atr_outcome], + outcome=final_outcomes[atr_outcome], ) item.ihook.pytest_runtest_logreport(report=final_report) @@ -72,15 +95,7 @@ def atr_get_failed_reports(terminalreporter: _pytest.terminal.TerminalReporter) return terminalreporter.getreports(_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED) -def _atr_do_retries(item: pytest.Item) -> TestStatus: - outcomes = RetryOutcomes( - PASSED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, - FAILED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, - SKIPPED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, - XFAIL=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, - XPASS=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, - ) - +def _atr_do_retries(item: pytest.Item, outcomes: RetryOutcomes) -> TestStatus: test_id = _get_test_id_from_item(item) while InternalTest.atr_should_retry(test_id): @@ -160,21 +175,21 @@ def atr_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.Te _atr_write_report_for_status( terminalreporter, - _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, - "failed", - PYTEST_STATUS.FAILED, - raw_summary_strings, - markedup_summary_strings, + status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, + status_text="failed", + report_outcome=PYTEST_STATUS.FAILED, + raw_strings=raw_summary_strings, + markedup_strings=markedup_summary_strings, color="red", ) _atr_write_report_for_status( terminalreporter, - _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, - "passed", - PYTEST_STATUS.PASSED, - raw_summary_strings, - markedup_summary_strings, + status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + status_text="passed", + report_outcome=PYTEST_STATUS.PASSED, + raw_strings=raw_summary_strings, + markedup_strings=markedup_summary_strings, color="green", ) @@ -268,3 +283,47 @@ def atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_r if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED: return (_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, "F", ("ATR FINAL STATUS: FAILED", {"red": True})) return None + + +def quarantine_atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_return_type: + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, + "q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}PASSED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, + "Q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}FAILED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, + "q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}SKIPPED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + ".", + ("QUARANTINED FINAL STATUS: PASSED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, + "F", + ("QUARANTINED FINAL STATUS: FAILED", {"blue": True}), + ) + return None + + +def quarantine_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.TerminalReporter): + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, []) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, []) + + # TODO: report list of fully failed quarantined tests, possibly inside the ATR report. diff --git a/ddtrace/contrib/pytest/_plugin_v2.py b/ddtrace/contrib/pytest/_plugin_v2.py index f1da8d2db11..f15373a776a 100644 --- a/ddtrace/contrib/pytest/_plugin_v2.py +++ b/ddtrace/contrib/pytest/_plugin_v2.py @@ -76,11 +76,16 @@ from ddtrace.contrib.pytest._atr_utils import atr_get_teststatus from ddtrace.contrib.pytest._atr_utils import atr_handle_retries from ddtrace.contrib.pytest._atr_utils import atr_pytest_terminal_summary_post_yield + from ddtrace.contrib.pytest._atr_utils import quarantine_atr_get_teststatus + from ddtrace.contrib.pytest._atr_utils import quarantine_pytest_terminal_summary_post_yield log = get_logger(__name__) _NODEID_REGEX = re.compile("^((?P.*)/(?P[^/]*?))::(?P.*?)$") +USER_PROPERTY_QUARANTINED = "dd_quarantined" +OUTCOME_QUARANTINED = "quarantined" +SKIPPED_BY_QUARANTINE_REASON = "Skipped by Datadog Quarantine" def _handle_itr_should_skip(item, test_id) -> bool: @@ -109,6 +114,19 @@ def _handle_itr_should_skip(item, test_id) -> bool: return False +def _handle_quarantine(item, test_id): + """Add a user property to identify quarantined tests, and mark them for skipping if quarantine is enabled in + skipping mode. + """ + is_quarantined = InternalTest.is_quarantined_test(test_id) + if is_quarantined: + # We add this information to user_properties to have it available in pytest_runtest_makereport(). + item.user_properties += [(USER_PROPERTY_QUARANTINED, True)] + + if InternalTestSession.should_skip_quarantined_tests(): + item.add_marker(pytest.mark.skip(reason=SKIPPED_BY_QUARANTINE_REASON)) + + def _start_collecting_coverage() -> ModuleCodeCollector.CollectInContext: coverage_collector = ModuleCodeCollector.CollectInContext() # TODO: don't depend on internal for telemetry @@ -321,6 +339,7 @@ def _pytest_runtest_protocol_pre_yield(item) -> t.Optional[ModuleCodeCollector.C InternalTest.start(test_id) + _handle_quarantine(item, test_id) _handle_itr_should_skip(item, test_id) item_will_skip = _pytest_marked_to_skip(item) or InternalTest.was_skipped_by_itr(test_id) @@ -457,6 +476,8 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome test_id = _get_test_id_from_item(item) + is_quarantined = InternalTest.is_quarantined_test(test_id) + test_outcome = _process_result(item, call, original_result) # A None value for test_outcome.status implies the test has not finished yet @@ -472,6 +493,11 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome if not InternalTest.is_finished(test_id): InternalTest.finish(test_id, test_outcome.status, test_outcome.skip_reason, test_outcome.exc_info) + if original_result.failed and is_quarantined: + # Ensure test doesn't count as failed for pytest's exit status logic + # (see ). + original_result.outcome = OUTCOME_QUARANTINED + # ATR and EFD retry tests only if their teardown succeeded to ensure the best chance the retry will succeed # NOTE: this mutates the original result's outcome if InternalTest.stash_get(test_id, "setup_failed") or InternalTest.stash_get(test_id, "teardown_failed"): @@ -480,7 +506,7 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome if InternalTestSession.efd_enabled() and InternalTest.efd_should_retry(test_id): return efd_handle_retries(test_id, item, call.when, original_result, test_outcome) if InternalTestSession.atr_is_enabled() and InternalTest.atr_should_retry(test_id): - return atr_handle_retries(test_id, item, call.when, original_result, test_outcome) + return atr_handle_retries(test_id, item, call.when, original_result, test_outcome, is_quarantined) @pytest.hookimpl(hookwrapper=True) @@ -538,6 +564,9 @@ def _pytest_terminal_summary_post_yield(terminalreporter, failed_reports_initial if _pytest_version_supports_atr() and InternalTestSession.atr_is_enabled(): atr_pytest_terminal_summary_post_yield(terminalreporter) + + quarantine_pytest_terminal_summary_post_yield(terminalreporter) + return @@ -577,7 +606,7 @@ def _pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if InternalTestSession.efd_enabled() and InternalTestSession.efd_has_failed_tests(): session.exitstatus = pytest.ExitCode.TESTS_FAILED - if InternalTestSession.atr_has_failed_tests() and InternalTestSession.atr_has_failed_tests(): + if InternalTestSession.atr_is_enabled() and InternalTestSession.atr_has_failed_tests(): session.exitstatus = pytest.ExitCode.TESTS_FAILED invoked_by_coverage_run_status = _is_coverage_invoked_by_coverage_run() @@ -615,7 +644,7 @@ def pytest_report_teststatus( return if _pytest_version_supports_atr() and InternalTestSession.atr_is_enabled(): - test_status = atr_get_teststatus(report) + test_status = atr_get_teststatus(report) or quarantine_atr_get_teststatus(report) if test_status is not None: return test_status @@ -624,6 +653,16 @@ def pytest_report_teststatus( if test_status is not None: return test_status + user_properties = getattr(report, "user_properties", []) + is_quarantined = (USER_PROPERTY_QUARANTINED, True) in user_properties + if is_quarantined: + if report.when == "teardown": + return (OUTCOME_QUARANTINED, "q", ("QUARANTINED", {"blue": True})) + else: + # Don't show anything for setup and call of quarantined tests, regardless of + # whether there were errors or not. + return ("", "", "") + @pytest.hookimpl(trylast=True) def pytest_ddtrace_get_item_module_name(item): diff --git a/ddtrace/contrib/pytest/_retry_utils.py b/ddtrace/contrib/pytest/_retry_utils.py index de68e7b7c51..6e38a2974c8 100644 --- a/ddtrace/contrib/pytest/_retry_utils.py +++ b/ddtrace/contrib/pytest/_retry_utils.py @@ -52,6 +52,9 @@ def _get_outcome_from_retry( # _initrequest() needs to be called first because the test has already executed once item._initrequest() + # Reset output capture across retries. + item._report_sections = [] + # Setup setup_call, setup_report = _retry_run_when(item, "setup", outcomes) if setup_report.outcome == outcomes.FAILED: @@ -80,7 +83,6 @@ def _get_outcome_from_retry( _outcome_status = TestStatus.SKIP elif call_report.outcome == outcomes.PASSED: _outcome_status = TestStatus.PASS - # Teardown does not happen if setup skipped if not setup_report.skipped: teardown_call, teardown_report = _retry_run_when(item, "teardown", outcomes) diff --git a/ddtrace/debugging/_debugger.py b/ddtrace/debugging/_debugger.py index 1c2429ba569..c36bb94fdfb 100644 --- a/ddtrace/debugging/_debugger.py +++ b/ddtrace/debugging/_debugger.py @@ -6,6 +6,7 @@ from pathlib import Path import sys import threading +import time from types import CodeType from types import FunctionType from types import ModuleType @@ -43,7 +44,6 @@ from ddtrace.debugging._signal.model import SignalState from ddtrace.debugging._uploader import LogsIntakeUploaderV1 from ddtrace.debugging._uploader import UploaderProduct -from ddtrace.internal import compat from ddtrace.internal.logger import get_logger from ddtrace.internal.metrics import Metrics from ddtrace.internal.module import ModuleHookType @@ -202,11 +202,11 @@ def _open_signals(self) -> None: signals.append(signal) # Save state on the wrapping context - self.set("start_time", compat.monotonic_ns()) + self.set("start_time", time.monotonic_ns()) self.set("signals", signals) def _close_signals(self, retval=None, exc_info=(None, None, None)) -> None: - end_time = compat.monotonic_ns() + end_time = time.monotonic_ns() signals = cast(Deque[Signal], self.get("signals")) while signals: # Open probe signals are ordered, with those that have created new diff --git a/ddtrace/debugging/_origin/span.py b/ddtrace/debugging/_origin/span.py index bd3744c20f2..9b592df2bde 100644 --- a/ddtrace/debugging/_origin/span.py +++ b/ddtrace/debugging/_origin/span.py @@ -2,6 +2,7 @@ from itertools import count from pathlib import Path import sys +import time # from threading import current_thread from types import FrameType @@ -24,7 +25,6 @@ # from ddtrace.debugging._signal.snapshot import Snapshot from ddtrace.debugging._signal.model import Signal from ddtrace.ext import EXIT_SPAN_TYPES -from ddtrace.internal import compat from ddtrace.internal import core from ddtrace.internal.packages import is_user_code from ddtrace.internal.safety import _isinstance @@ -170,7 +170,7 @@ def __enter__(self): # span.set_tag_str("_dd.code_origin.frames.0.snapshot_id", snapshot.uuid) # self.set("context", context) - # self.set("start_time", compat.monotonic_ns()) + # self.set("start_time", time.monotonic_ns()) return self @@ -181,7 +181,7 @@ def _close_signal(self, retval=None, exc_info=(None, None, None)): # No snapshot was created return - signal.do_exit(retval, exc_info, compat.monotonic_ns() - self.get("start_time")) + signal.do_exit(retval, exc_info, time.monotonic_ns() - self.get("start_time")) def __return__(self, retval): self._close_signal(retval=retval) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index aaeaa59f1d7..49722e51e41 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -4,6 +4,7 @@ from http.client import RemoteDisconnected import json from json import JSONDecodeError +import os import socket import typing as t from uuid import uuid4 @@ -38,6 +39,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility._internal_item_ids import InternalTestId from ddtrace.internal.test_visibility.coverage_lines import CoverageLines +from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.http import ConnectionType from ddtrace.internal.utils.http import Response from ddtrace.internal.utils.http import get_connection @@ -87,6 +89,12 @@ class EarlyFlakeDetectionSettings: faulty_session_threshold: int = 30 +@dataclasses.dataclass(frozen=True) +class QuarantineSettings: + enabled: bool = False + skip_quarantined_tests: bool = False + + @dataclasses.dataclass(frozen=True) class TestVisibilityAPISettings: __test__ = False @@ -96,6 +104,7 @@ class TestVisibilityAPISettings: itr_enabled: bool = False flaky_test_retries_enabled: bool = False early_flake_detection: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings) + quarantine: QuarantineSettings = dataclasses.field(default_factory=QuarantineSettings) @dataclasses.dataclass(frozen=True) @@ -359,7 +368,9 @@ def fetch_settings(self) -> TestVisibilityAPISettings: skipping_enabled = attributes["tests_skipping"] require_git = attributes["require_git"] itr_enabled = attributes["itr_enabled"] - flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] + flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] or asbool( + os.getenv("_DD_TEST_FORCE_ENABLE_ATR") + ) if attributes["early_flake_detection"]["enabled"]: early_flake_detection = EarlyFlakeDetectionSettings( @@ -372,6 +383,14 @@ def fetch_settings(self) -> TestVisibilityAPISettings: ) else: early_flake_detection = EarlyFlakeDetectionSettings() + + quarantine = QuarantineSettings( + enabled=attributes.get("quarantine", {}).get("enabled", False) + or asbool(os.getenv("_DD_TEST_FORCE_ENABLE_QUARANTINE")), + skip_quarantined_tests=attributes.get("quarantine", {}).get("skip_quarantined_tests", False) + or asbool(os.getenv("_DD_TEST_SKIP_QUARANTINED_TESTS")), + ) + except KeyError: record_api_request_error(metric_names.error, ERROR_TYPES.UNKNOWN) raise @@ -383,6 +402,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings: itr_enabled=itr_enabled, flaky_test_retries_enabled=flaky_test_retries_enabled, early_flake_detection=early_flake_detection, + quarantine=quarantine, ) record_settings_response( @@ -392,6 +412,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings: itr_enabled=api_settings.itr_enabled, flaky_test_retries_enabled=api_settings.flaky_test_retries_enabled, early_flake_detection_enabled=api_settings.early_flake_detection.enabled, + quarantine_enabled=api_settings.quarantine.enabled, ) return api_settings diff --git a/ddtrace/internal/ci_visibility/api/_base.py b/ddtrace/internal/ci_visibility/api/_base.py index f1e2cd2b3b0..1e56d6b00b6 100644 --- a/ddtrace/internal/ci_visibility/api/_base.py +++ b/ddtrace/internal/ci_visibility/api/_base.py @@ -25,6 +25,7 @@ from ddtrace.ext.test_visibility.api import TestSourceFileInfo from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility.api._coverage_data import TestVisibilityCoverageData from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import EVENT_TYPE @@ -71,6 +72,7 @@ class TestVisibilitySessionSettings: coverage_enabled: bool = False efd_settings: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings) atr_settings: AutoTestRetriesSettings = dataclasses.field(default_factory=AutoTestRetriesSettings) + quarantine_settings: QuarantineSettings = dataclasses.field(default_factory=QuarantineSettings) def __post_init__(self): if not isinstance(self.tracer, Tracer): @@ -207,6 +209,12 @@ def _finish_span(self, override_finish_time: Optional[float] = None) -> None: if self._session_settings.atr_settings is not None and self._session_settings.atr_settings.enabled: self._set_atr_tags() + if ( + self._session_settings.quarantine_settings is not None + and self._session_settings.quarantine_settings.enabled + ): + self._set_quarantine_tags() + # Allow items to potentially overwrite default and hierarchy tags. self._set_item_tags() self._set_span_tags() @@ -272,6 +280,10 @@ def _set_atr_tags(self) -> None: """ATR tags are only set at the test level""" pass + def _set_quarantine_tags(self) -> None: + """Quarantine tags are only set at the test or session level""" + pass + def _set_span_tags(self): """This is effectively a callback method for exceptional cases where the item span needs to be modified directly by the class diff --git a/ddtrace/internal/ci_visibility/api/_session.py b/ddtrace/internal/ci_visibility/api/_session.py index 5267a345c0a..44f8aef38f5 100644 --- a/ddtrace/internal/ci_visibility/api/_session.py +++ b/ddtrace/internal/ci_visibility/api/_session.py @@ -15,6 +15,7 @@ from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_EFD_ABORT_REASON from ddtrace.internal.ci_visibility.constants import TEST_EFD_ENABLED +from ddtrace.internal.ci_visibility.constants import TEST_SESSION_QUARANTINE_ENABLED from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.events import record_event_created from ddtrace.internal.ci_visibility.telemetry.events import record_event_finished @@ -71,6 +72,9 @@ def _set_efd_tags(self): elif self.efd_is_faulty_session(): self.set_tag(TEST_EFD_ABORT_REASON, "faulty") + def _set_quarantine_tags(self): + self.set_tag(TEST_SESSION_QUARANTINE_ENABLED, True) + def _set_itr_tags(self, itr_enabled: bool) -> None: """Set session-level tags based in ITR enablement status""" super()._set_itr_tags(itr_enabled) @@ -178,6 +182,8 @@ def atr_has_failed_tests(self): for _module in self._children.values(): for _suite in _module._children.values(): for _test in _suite._children.values(): + if _test.is_quarantined(): + continue if _test.atr_has_retries() and _test.atr_get_final_status() == TestStatus.FAIL: return True return False diff --git a/ddtrace/internal/ci_visibility/api/_suite.py b/ddtrace/internal/ci_visibility/api/_suite.py index 2eca7316cd0..ba24f82c05c 100644 --- a/ddtrace/internal/ci_visibility/api/_suite.py +++ b/ddtrace/internal/ci_visibility/api/_suite.py @@ -57,7 +57,7 @@ def finish( super().finish(force=force, override_status=override_status, override_finish_time=override_finish_time) def finish_itr_skipped(self) -> None: - """Suites should only count themselves as ITR-skipped of all children are ITR skipped""" + """Suites should only count themselves as ITR-skipped if all children are ITR skipped""" log.debug("Finishing CI Visibility suite %s as ITR skipped", self) for child in self._children.values(): if not (child.is_finished() and child.is_itr_skipped()): diff --git a/ddtrace/internal/ci_visibility/api/_test.py b/ddtrace/internal/ci_visibility/api/_test.py index c63d9753eb8..0f8a2efd41d 100644 --- a/ddtrace/internal/ci_visibility/api/_test.py +++ b/ddtrace/internal/ci_visibility/api/_test.py @@ -21,7 +21,9 @@ from ddtrace.internal.ci_visibility.constants import BENCHMARK from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_EFD_ABORT_REASON +from ddtrace.internal.ci_visibility.constants import TEST_HAS_FAILED_ALL_RETRIES from ddtrace.internal.ci_visibility.constants import TEST_IS_NEW +from ddtrace.internal.ci_visibility.constants import TEST_IS_QUARANTINED from ddtrace.internal.ci_visibility.constants import TEST_IS_RETRY from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.events import record_event_created_test @@ -55,6 +57,7 @@ def __init__( is_atr_retry: bool = False, resource: Optional[str] = None, is_new: bool = False, + is_quarantined: bool = False, ): self._parameters = parameters super().__init__( @@ -74,6 +77,7 @@ def __init__( self.set_tag(test.PARAMETERS, parameters) self._is_new = is_new + self._is_quarantined = is_quarantined self._efd_is_retry = is_efd_retry self._efd_retries: List[TestVisibilityTest] = [] @@ -126,6 +130,10 @@ def _set_atr_tags(self) -> None: if self._atr_is_retry: self.set_tag(TEST_IS_RETRY, self._atr_is_retry) + def _set_quarantine_tags(self) -> None: + if self._is_quarantined: + self.set_tag(TEST_IS_QUARANTINED, self._is_quarantined) + def _set_span_tags(self) -> None: """This handles setting tags that can't be properly stored in self._tags @@ -149,6 +157,7 @@ def _telemetry_record_event_finished(self): is_new=self.is_new(), is_retry=self._efd_is_retry or self._atr_is_retry, early_flake_detection_abort_reason=self._efd_abort_reason, + is_quarantined=self.is_quarantined(), is_rum=self._is_rum(), browser_driver=self._get_browser_driver(), ) @@ -236,6 +245,11 @@ def is_new(self): # decisions) return self._is_new and (self._parameters is None) + def is_quarantined(self): + return self._session_settings.quarantine_settings.enabled and ( + self._is_quarantined or self.get_tag(TEST_IS_QUARANTINED) + ) + # # EFD (Early Flake Detection) functionality # @@ -360,6 +374,7 @@ def _atr_make_retry_test(self): codeowners=self._codeowners, source_file_info=self._source_file_info, initial_tags=self._tags, + is_quarantined=self.is_quarantined(), is_atr_retry=True, ) retry_test.parent = self.parent @@ -406,7 +421,13 @@ def atr_start_retry(self, retry_number: int): self._atr_get_retry_test(retry_number).start() def atr_finish_retry(self, retry_number: int, status: TestStatus, exc_info: Optional[TestExcInfo] = None): - self._atr_get_retry_test(retry_number).finish_test(status, exc_info=exc_info) + retry_test = self._atr_get_retry_test(retry_number) + + if retry_number >= self._session_settings.atr_settings.max_retries: + if self.atr_get_final_status() == TestStatus.FAIL and self.is_quarantined(): + retry_test.set_tag(TEST_HAS_FAILED_ALL_RETRIES, True) + + retry_test.finish_test(status, exc_info=exc_info) def atr_get_final_status(self) -> TestStatus: if self._status in [TestStatus.PASS, TestStatus.SKIP]: diff --git a/ddtrace/internal/ci_visibility/constants.py b/ddtrace/internal/ci_visibility/constants.py index 7ace37b9424..5a14111bc3b 100644 --- a/ddtrace/internal/ci_visibility/constants.py +++ b/ddtrace/internal/ci_visibility/constants.py @@ -48,6 +48,7 @@ SETTING_ENDPOINT = "/api/v2/libraries/tests/services/setting" SKIPPABLE_ENDPOINT = "/api/v2/ci/tests/skippable" UNIQUE_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests" +DETAILED_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests/detailed" # Intelligent Test Runner constants ITR_UNSKIPPABLE_REASON = "datadog_itr_unskippable" @@ -82,5 +83,9 @@ class REQUESTS_MODE(IntEnum): # EFD and auto retries TEST_IS_NEW = "test.is_new" TEST_IS_RETRY = "test.is_retry" +TEST_IS_QUARANTINED = "test.quarantine.is_quarantined" TEST_EFD_ABORT_REASON = "test.early_flake.abort_reason" TEST_EFD_ENABLED = "test.early_flake.enabled" +TEST_HAS_FAILED_ALL_RETRIES = "test.has_failed_all_retries" + +TEST_SESSION_QUARANTINE_ENABLED = "test_session.quarantine.enabled" diff --git a/ddtrace/internal/ci_visibility/recorder.py b/ddtrace/internal/ci_visibility/recorder.py index 0046c21be15..609475506d3 100644 --- a/ddtrace/internal/ci_visibility/recorder.py +++ b/ddtrace/internal/ci_visibility/recorder.py @@ -39,6 +39,7 @@ from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings from ddtrace.internal.ci_visibility._api_client import EVPProxyTestVisibilityAPIClient from ddtrace.internal.ci_visibility._api_client import ITRData +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility._api_client import _TestVisibilityAPIClientBase from ddtrace.internal.ci_visibility.api._module import TestVisibilityModule @@ -446,6 +447,22 @@ def is_atr_enabled(cls): os.getenv("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED", default=True) ) + @classmethod + def is_quarantine_enabled(cls): + if cls._instance is None: + return False + return cls._instance._api_settings.quarantine.enabled and asbool( + os.getenv("DD_TEST_QUARANTINE_ENABLED", default=True) + ) + + @classmethod + def should_skip_quarantined_tests(cls): + if cls._instance is None: + return False + return cls._instance._api_settings.quarantine.skip_quarantined_tests and asbool( + os.getenv("DD_TEST_QUARANTINE_ENABLED", default=True) + ) + @classmethod def should_collect_coverage(cls): return cls._instance._api_settings.coverage_enabled or asbool( @@ -546,11 +563,13 @@ def enable(cls, tracer=None, config=None, service=None): "Final settings: coverage collection: %s, " "test skipping: %s, " "Early Flake Detection: %s, " - "Auto Test Retries: %s", + "Auto Test Retries: %s, " + "Quarantine: %s", cls._instance._collect_coverage_enabled, CIVisibility.test_skipping_enabled(), CIVisibility.is_efd_enabled(), CIVisibility.is_atr_enabled(), + CIVisibility.is_quarantine_enabled(), ) @classmethod @@ -821,6 +840,17 @@ def get_atr_api_settings(cls) -> Optional[AutoTestRetriesSettings]: return None + @classmethod + def get_quarantine_api_settings(cls) -> Optional[QuarantineSettings]: + if not cls.enabled: + error_msg = "CI Visibility is not enabled" + log.warning(error_msg) + raise CIVisibilityError(error_msg) + instance = cls.get_instance() + if instance is None or instance._api_settings is None: + return None + return instance._api_settings.quarantine + @classmethod def get_workspace_path(cls) -> Optional[str]: if not cls.enabled: @@ -900,6 +930,15 @@ def is_unique_test(cls, test_id: Union[TestId, InternalTestId]) -> bool: return test_id in instance._unique_test_ids + @classmethod + def is_quarantined(cls, test_id: Union[TestId, InternalTestId]) -> bool: + instance = cls.get_instance() + if instance is None: + return False + + # TODO: retrieve this information from the API, once it is available in the backend. + return False + def _requires_civisibility_enabled(func): def wrapper(*args, **kwargs): @@ -939,6 +978,10 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs): if atr_api_settings is None or not CIVisibility.is_atr_enabled(): atr_api_settings = AutoTestRetriesSettings() + quarantine_api_settings = CIVisibility.get_quarantine_api_settings() + if quarantine_api_settings is None or not CIVisibility.is_quarantine_enabled(): + quarantine_api_settings = QuarantineSettings() + session_settings = TestVisibilitySessionSettings( tracer=tracer, test_service=test_service, @@ -960,6 +1003,7 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs): coverage_enabled=CIVisibility.should_collect_coverage(), efd_settings=efd_api_settings, atr_settings=atr_api_settings, + quarantine_settings=quarantine_api_settings, ) session = TestVisibilitySession( @@ -1003,6 +1047,12 @@ def _on_session_should_collect_coverage() -> bool: return CIVisibility.should_collect_coverage() +@_requires_civisibility_enabled +def _on_session_should_skip_quarantined_tests() -> bool: + log.debug("Handling should skip quarantined tests") + return CIVisibility.should_skip_quarantined_tests() + + @_requires_civisibility_enabled def _on_session_get_codeowners() -> Optional[Codeowners]: log.debug("Getting codeowners") @@ -1064,6 +1114,11 @@ def _register_session_handlers(): "is_test_skipping_enabled", ) core.on("test_visibility.session.set_covered_lines_pct", _on_session_set_covered_lines_pct) + core.on( + "test_visibility.session.should_skip_quarantined_tests", + _on_session_should_skip_quarantined_tests, + "should_skip_quarantined_tests", + ) @_requires_civisibility_enabled @@ -1150,6 +1205,11 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): else: is_new = False + if CIVisibility.is_quarantine_enabled(): + is_quarantined = CIVisibility.is_quarantined(discover_args.test_id) + else: + is_quarantined = False + suite.add_child( discover_args.test_id, TestVisibilityTest( @@ -1160,6 +1220,7 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): source_file_info=discover_args.source_file_info, resource=discover_args.resource, is_new=is_new, + is_quarantined=is_quarantined, ), ) @@ -1170,6 +1231,12 @@ def _on_is_new_test(test_id: Union[TestId, InternalTestId]) -> bool: return CIVisibility.get_test_by_id(test_id).is_new() +@_requires_civisibility_enabled +def _on_is_quarantined_test(test_id: Union[TestId, InternalTestId]) -> bool: + log.debug("Handling is quarantined test for test %s", test_id) + return CIVisibility.get_test_by_id(test_id).is_quarantined() + + @_requires_civisibility_enabled def _on_start_test(test_id: TestId): log.debug("Handling start for test id %s", test_id) @@ -1215,6 +1282,7 @@ def _register_test_handlers(): log.debug("Registering test handlers") core.on("test_visibility.test.discover", _on_discover_test) core.on("test_visibility.test.is_new", _on_is_new_test, "is_new") + core.on("test_visibility.test.is_quarantined", _on_is_quarantined_test, "is_quarantined") core.on("test_visibility.test.start", _on_start_test) core.on("test_visibility.test.finish", _on_finish_test) core.on("test_visibility.test.set_parameters", _on_set_test_parameters) diff --git a/ddtrace/internal/ci_visibility/telemetry/events.py b/ddtrace/internal/ci_visibility/telemetry/events.py index be39c8079cf..34c603c3b03 100644 --- a/ddtrace/internal/ci_visibility/telemetry/events.py +++ b/ddtrace/internal/ci_visibility/telemetry/events.py @@ -50,7 +50,6 @@ def _record_event( log.debug("has_codeowners tag can only be set for sessions, but event type is %s", event_type) if is_unsupported_ci and event_type != EVENT_TYPES.SESSION: log.debug("unsupported_ci tag can only be set for sessions, but event type is %s", event_type) - if early_flake_detection_abort_reason and ( event_type not in [EVENT_TYPES.SESSION] or event != EVENTS_TELEMETRY.FINISHED ): @@ -151,6 +150,7 @@ def record_event_finished_test( is_rum: bool = False, browser_driver: Optional[str] = None, is_benchmark: bool = False, + is_quarantined: bool = False, ): log.debug( "Recording test event finished: test_framework=%s" @@ -159,7 +159,8 @@ def record_event_finished_test( ", early_flake_detection_abort_reason=%s" ", is_rum=%s" ", browser_driver=%s" - ", is_benchmark=%s", + ", is_benchmark=%s" + ", is_quarantined=%s", test_framework, is_new, is_retry, @@ -167,6 +168,7 @@ def record_event_finished_test( is_rum, browser_driver, is_benchmark, + is_quarantined, ) tags: List[Tuple[str, str]] = [("event_type", EVENT_TYPES.TEST)] @@ -185,5 +187,7 @@ def record_event_finished_test( tags.append(("browser_driver", browser_driver)) if early_flake_detection_abort_reason is not None: tags.append(("early_flake_detection_abort_reason", early_flake_detection_abort_reason)) + if is_quarantined: + tags.append(("is_quarantined", "true")) telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) diff --git a/ddtrace/internal/ci_visibility/telemetry/git.py b/ddtrace/internal/ci_visibility/telemetry/git.py index 761ecc62a00..faf01621cde 100644 --- a/ddtrace/internal/ci_visibility/telemetry/git.py +++ b/ddtrace/internal/ci_visibility/telemetry/git.py @@ -52,6 +52,7 @@ def record_settings_response( itr_enabled: Optional[bool] = False, flaky_test_retries_enabled: Optional[bool] = False, early_flake_detection_enabled: Optional[bool] = False, + quarantine_enabled: Optional[bool] = False, ) -> None: log.debug( "Recording settings telemetry:" @@ -82,6 +83,8 @@ def record_settings_response( response_tags.append(("flaky_test_retries_enabled", "true")) if early_flake_detection_enabled: response_tags.append(("early_flake_detection_enabled", "true")) + if quarantine_enabled: + response_tags.append(("quarantine_enabled", "true")) if response_tags: telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags)) diff --git a/ddtrace/internal/compat.py b/ddtrace/internal/compat.py index 457618dc393..7f00043f049 100644 --- a/ddtrace/internal/compat.py +++ b/ddtrace/internal/compat.py @@ -122,41 +122,6 @@ def is_integer(obj): return isinstance(obj, int) and not isinstance(obj, bool) -try: - from time import time_ns -except ImportError: - from time import time as _time - - def time_ns(): - # type: () -> int - return int(_time() * 10e5) * 1000 - - -try: - from time import monotonic -except ImportError: - from ddtrace.vendor.monotonic import monotonic - - -try: - from time import monotonic_ns -except ImportError: - - def monotonic_ns(): - # type: () -> int - return int(monotonic() * 1e9) - - -try: - from time import process_time_ns -except ImportError: - from time import clock as _process_time # type: ignore[attr-defined] - - def process_time_ns(): - # type: () -> int - return int(_process_time() * 1e9) - - main_thread = threading.main_thread() diff --git a/ddtrace/internal/opentelemetry/span.py b/ddtrace/internal/opentelemetry/span.py index bd2b0dfe83b..15f497a358e 100644 --- a/ddtrace/internal/opentelemetry/span.py +++ b/ddtrace/internal/opentelemetry/span.py @@ -1,3 +1,4 @@ +from time import time_ns import traceback from typing import TYPE_CHECKING @@ -15,7 +16,6 @@ from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE from ddtrace.constants import SPAN_KIND -from ddtrace.internal.compat import time_ns from ddtrace.internal.logger import get_logger from ddtrace.internal.utils.formats import flatten_key_value from ddtrace.internal.utils.formats import is_sequence diff --git a/ddtrace/internal/rate_limiter.py b/ddtrace/internal/rate_limiter.py index 981701cd51b..0a97a6a7abc 100644 --- a/ddtrace/internal/rate_limiter.py +++ b/ddtrace/internal/rate_limiter.py @@ -4,6 +4,7 @@ from dataclasses import field import random import threading +import time from typing import Any # noqa:F401 from typing import Callable # noqa:F401 from typing import Optional # noqa:F401 @@ -11,8 +12,6 @@ from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate -from ..internal import compat - class RateLimiter(object): """ @@ -49,7 +48,7 @@ def __init__(self, rate_limit: int, time_window: float = 1e9): self.tokens = rate_limit # type: float self.max_tokens = rate_limit - self.last_update_ns = compat.monotonic_ns() + self.last_update_ns = time.monotonic_ns() self.current_window_ns = 0 # type: float self.tokens_allowed = 0 @@ -77,7 +76,7 @@ def is_allowed(self, timestamp_ns: Optional[int] = None) -> bool: # rate limits are tested and mocked in pytest so we need to compute the timestamp here # (or move the unit tests to rust) - timestamp_ns = timestamp_ns or compat.monotonic_ns() + timestamp_ns = timestamp_ns or time.monotonic_ns() allowed = self._is_allowed(timestamp_ns) # Update counts used to determine effective rate self._update_rate_counts(allowed, timestamp_ns) @@ -213,7 +212,7 @@ class BudgetRateLimiterWithJitter: call_once: bool = False budget: float = field(init=False) max_budget: float = field(init=False) - last_time: float = field(init=False, default_factory=compat.monotonic) + last_time: float = field(init=False, default_factory=time.monotonic) _lock: threading.Lock = field(init=False, default_factory=threading.Lock) def __post_init__(self): @@ -229,7 +228,7 @@ def limit(self, f: Optional[Callable[..., Any]] = None, *args: Any, **kwargs: An """Make rate-limited calls to a function with the given arguments.""" should_call = False with self._lock: - now = compat.monotonic() + now = time.monotonic() self.budget += self.limit_rate * (now - self.last_time) * (0.5 + random.random()) # jitter should_call = self.budget >= 1.0 if self.budget > self.max_budget: diff --git a/ddtrace/internal/test_visibility/api.py b/ddtrace/internal/test_visibility/api.py index 84f559a4701..4c11135b76d 100644 --- a/ddtrace/internal/test_visibility/api.py +++ b/ddtrace/internal/test_visibility/api.py @@ -116,6 +116,20 @@ def is_test_skipping_enabled() -> bool: return _is_test_skipping_enabled + @staticmethod + @_catch_and_log_exceptions + def should_skip_quarantined_tests() -> bool: + log.debug("Checking if quarantined tests should be skipped") + + should_skip = bool( + core.dispatch_with_results( + "test_visibility.session.should_skip_quarantined_tests" + ).should_skip_quarantined_tests.value + ) + log.debug("Quarantined tests should be skipped: %s", should_skip) + + return should_skip + @staticmethod @_catch_and_log_exceptions def set_covered_lines_pct(coverage_pct: float): @@ -175,6 +189,16 @@ def is_new_test(item_id: InternalTestId) -> bool: log.debug("Test %s is new: %s", item_id, is_new) return is_new + @staticmethod + @_catch_and_log_exceptions + def is_quarantined_test(item_id: InternalTestId) -> bool: + log.debug("Checking if test %s is quarantined", item_id) + is_quarantined = bool( + core.dispatch_with_results("test_visibility.test.is_quarantined", (item_id,)).is_quarantined.value + ) + log.debug("Test %s is quarantined: %s", item_id, is_quarantined) + return is_quarantined + class OverwriteAttributesArgs(NamedTuple): test_id: InternalTestId name: t.Optional[str] = None diff --git a/ddtrace/internal/utils/time.py b/ddtrace/internal/utils/time.py index 1993b9dd21c..6be4946d803 100644 --- a/ddtrace/internal/utils/time.py +++ b/ddtrace/internal/utils/time.py @@ -1,9 +1,9 @@ from datetime import datetime +import time from types import TracebackType from typing import Optional from typing import Type # noqa:F401 -from ddtrace.internal import compat from ddtrace.internal.logger import get_logger @@ -46,7 +46,7 @@ def __init__(self) -> None: def start(self): # type: () -> StopWatch """Starts the watch.""" - self._started_at = compat.monotonic() + self._started_at = time.monotonic() return self def elapsed(self) -> float: @@ -59,7 +59,7 @@ def elapsed(self) -> float: if self._started_at is None: raise RuntimeError("Can not get the elapsed time of a stopwatch" " if it has not been started/stopped") if self._stopped_at is None: - now = compat.monotonic() + now = time.monotonic() else: now = self._stopped_at return now - self._started_at @@ -81,7 +81,7 @@ def stop(self): """Stops the watch.""" if self._started_at is None: raise RuntimeError("Can not stop a stopwatch that has not been" " started") - self._stopped_at = compat.monotonic() + self._stopped_at = time.monotonic() return self @@ -89,7 +89,7 @@ class HourGlass(object): """An implementation of an hourglass.""" def __init__(self, duration: float) -> None: - t = compat.monotonic() + t = time.monotonic() self._duration = duration self._started_at = t - duration @@ -99,7 +99,7 @@ def __init__(self, duration: float) -> None: def turn(self) -> None: """Turn the hourglass.""" - t = compat.monotonic() + t = time.monotonic() top_0 = self._end_at - self._started_at bottom = self._duration - top_0 + min(t - self._started_at, top_0) @@ -119,7 +119,7 @@ def _trickled(self): def _trickling(self): # type: () -> bool - if compat.monotonic() < self._end_at: + if time.monotonic() < self._end_at: return True # No longer trickling, so we change state diff --git a/ddtrace/profiling/collector/_lock.py b/ddtrace/profiling/collector/_lock.py index 4ee0e692fac..6dedf3295f7 100644 --- a/ddtrace/profiling/collector/_lock.py +++ b/ddtrace/profiling/collector/_lock.py @@ -4,13 +4,13 @@ import abc import os.path import sys +import time import types import typing import wrapt from ddtrace._trace.tracer import Tracer -from ddtrace.internal import compat from ddtrace.internal.datadog.profiling import ddup from ddtrace.internal.logger import get_logger from ddtrace.profiling import _threading @@ -117,12 +117,12 @@ def _acquire(self, inner_func, *args, **kwargs): if not self._self_capture_sampler.capture(): return inner_func(*args, **kwargs) - start = compat.monotonic_ns() + start = time.monotonic_ns() try: return inner_func(*args, **kwargs) finally: try: - end = self._self_acquired_at = compat.monotonic_ns() + end = self._self_acquired_at = time.monotonic_ns() thread_id, thread_name = _current_thread() task_id, task_name, task_frame = _task.get_task(thread_id) self._maybe_update_self_name() @@ -185,7 +185,7 @@ def _release(self, inner_func, *args, **kwargs): try: if hasattr(self, "_self_acquired_at"): try: - end = compat.monotonic_ns() + end = time.monotonic_ns() thread_id, thread_name = _current_thread() task_id, task_name, task_frame = _task.get_task(thread_id) lock_name = ( diff --git a/ddtrace/profiling/collector/memalloc.py b/ddtrace/profiling/collector/memalloc.py index 62d4b059214..549132ed244 100644 --- a/ddtrace/profiling/collector/memalloc.py +++ b/ddtrace/profiling/collector/memalloc.py @@ -3,6 +3,7 @@ from math import ceil import os import threading +import time import typing # noqa:F401 from typing import Optional @@ -12,7 +13,6 @@ except ImportError: _memalloc = None # type: ignore[assignment] -from ddtrace.internal import compat from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling import _threading from ddtrace.profiling import collector @@ -189,7 +189,7 @@ def collect(self): if thread_id in thread_id_ignore_set: continue handle = ddup.SampleHandle() - handle.push_monotonic_ns(compat.monotonic_ns()) + handle.push_monotonic_ns(time.monotonic_ns()) handle.push_alloc(int((ceil(size) * alloc_count) / count), count) # Roundup to help float precision handle.push_threadinfo( thread_id, _threading.get_thread_native_id(thread_id), _threading.get_thread_name(thread_id) diff --git a/ddtrace/profiling/collector/stack.pyx b/ddtrace/profiling/collector/stack.pyx index c7ba1ec3e83..46b24e39c33 100644 --- a/ddtrace/profiling/collector/stack.pyx +++ b/ddtrace/profiling/collector/stack.pyx @@ -4,13 +4,13 @@ from __future__ import absolute_import from itertools import chain import logging import sys +import time import typing from ddtrace.internal._unpatched import _threading as ddtrace_threading from ddtrace._trace import context from ddtrace._trace import span as ddspan from ddtrace._trace.tracer import Tracer -from ddtrace.internal import compat from ddtrace.internal._threads import periodic_threads from ddtrace.internal.datadog.profiling import ddup from ddtrace.internal.datadog.profiling import stack_v2 @@ -131,10 +131,10 @@ ELSE: cdef stdint.int64_t _last_process_time def __init__(self): - self._last_process_time = compat.process_time_ns() + self._last_process_time = time.process_time_ns() def __call__(self, pthread_ids): - current_process_time = compat.process_time_ns() + current_process_time = time.process_time_ns() cpu_time = current_process_time - self._last_process_time self._last_process_time = current_process_time # Spread the consumed CPU time on all threads. @@ -524,7 +524,7 @@ class StackCollector(collector.PeriodicCollector): def _init(self): # type: (...) -> None self._thread_time = _ThreadTime() - self._last_wall_time = compat.monotonic_ns() + self._last_wall_time = time.monotonic_ns() if self.tracer is not None: self._thread_span_links = _ThreadSpanLinks() link_span = stack_v2.link_span if self._stack_collector_v2_enabled else self._thread_span_links.link_span @@ -569,7 +569,7 @@ class StackCollector(collector.PeriodicCollector): def collect(self): # Compute wall time - now = compat.monotonic_ns() + now = time.monotonic_ns() wall_time = now - self._last_wall_time self._last_wall_time = now all_events = [] @@ -587,7 +587,7 @@ class StackCollector(collector.PeriodicCollector): now_ns=now, ) - used_wall_time_ns = compat.monotonic_ns() - now + used_wall_time_ns = time.monotonic_ns() - now self.interval = self._compute_new_interval(used_wall_time_ns) if self._stack_collector_v2_enabled: diff --git a/ddtrace/profiling/scheduler.py b/ddtrace/profiling/scheduler.py index e8aafe7a63b..98ab424c42b 100644 --- a/ddtrace/profiling/scheduler.py +++ b/ddtrace/profiling/scheduler.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- import logging +import time from typing import Any # noqa F401 from typing import Callable from typing import Dict # noqa F401 @@ -9,7 +10,6 @@ import ddtrace from ddtrace._trace.tracer import Tracer -from ddtrace.internal import compat from ddtrace.internal import periodic from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling import _traceback @@ -49,7 +49,7 @@ def _start_service(self): """Start the scheduler.""" LOG.debug("Starting scheduler") super(Scheduler, self)._start_service() - self._last_export = compat.time_ns() + self._last_export = time.time_ns() LOG.debug("Scheduler started") def flush(self): @@ -68,14 +68,14 @@ def flush(self): # These are only used by the Python uploader, but set them here to keep logs/etc # consistent for now start = self._last_export - self._last_export = compat.time_ns() + self._last_export = time.time_ns() return events: EventsType = {} if self.recorder: events = self.recorder.reset() start = self._last_export - self._last_export = compat.time_ns() + self._last_export = time.time_ns() if self.exporters: for exp in self.exporters: try: @@ -90,11 +90,11 @@ def flush(self): def periodic(self): # type: (...) -> None - start_time = compat.monotonic() + start_time = time.monotonic() try: self.flush() finally: - self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) + self.interval = max(0, self._configured_interval - (time.monotonic() - start_time)) class ServerlessScheduler(Scheduler): @@ -119,7 +119,7 @@ def __init__(self, *args, **kwargs): def periodic(self): # type: (...) -> None # Check both the number of intervals and time frame to be sure we don't flush, e.g., empty profiles - if self._profiled_intervals >= self.FLUSH_AFTER_INTERVALS and (compat.time_ns() - self._last_export) >= ( + if self._profiled_intervals >= self.FLUSH_AFTER_INTERVALS and (time.time_ns() - self._last_export) >= ( self.FORCED_INTERVAL * self.FLUSH_AFTER_INTERVALS ): try: diff --git a/ddtrace/vendor/__init__.py b/ddtrace/vendor/__init__.py index 1b9596e82da..c74ff435562 100644 --- a/ddtrace/vendor/__init__.py +++ b/ddtrace/vendor/__init__.py @@ -26,19 +26,6 @@ removed unnecessary compat utils -monotonic ---------- - -Website: https://pypi.org/project/monotonic/ -Source: https://github.com/atdt/monotonic -Version: 1.5 -License: Apache License 2.0 - -Notes: - The source `monotonic.py` was added as `monotonic/__init__.py` - - No other changes were made - debtcollector ------------- @@ -107,7 +94,7 @@ yacc.py and lex.py files here. Didn't copy: cpp.py, ctokens.py, ygen.py (didn't see them used) - + jsonpath-ng --------- diff --git a/ddtrace/vendor/monotonic/__init__.py b/ddtrace/vendor/monotonic/__init__.py deleted file mode 100644 index 4ad147bae80..00000000000 --- a/ddtrace/vendor/monotonic/__init__.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -""" - monotonic - ~~~~~~~~~ - - This module provides a ``monotonic()`` function which returns the - value (in fractional seconds) of a clock which never goes backwards. - - On Python 3.3 or newer, ``monotonic`` will be an alias of - ``time.monotonic`` from the standard library. On older versions, - it will fall back to an equivalent implementation: - - +-------------+----------------------------------------+ - | Linux, BSD | ``clock_gettime(3)`` | - +-------------+----------------------------------------+ - | Windows | ``GetTickCount`` or ``GetTickCount64`` | - +-------------+----------------------------------------+ - | OS X | ``mach_absolute_time`` | - +-------------+----------------------------------------+ - - If no suitable implementation exists for the current platform, - attempting to import this module (or to import from it) will - cause a ``RuntimeError`` exception to be raised. - - - Copyright 2014, 2015, 2016 Ori Livneh - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -""" -import time - - -__all__ = ('monotonic',) - - -try: - monotonic = time.monotonic -except AttributeError: - import ctypes - import ctypes.util - import os - import sys - import threading - try: - if sys.platform == 'darwin': # OS X, iOS - # See Technical Q&A QA1398 of the Mac Developer Library: - # - libc = ctypes.CDLL('/usr/lib/libc.dylib', use_errno=True) - - class mach_timebase_info_data_t(ctypes.Structure): - """System timebase info. Defined in .""" - _fields_ = (('numer', ctypes.c_uint32), - ('denom', ctypes.c_uint32)) - - mach_absolute_time = libc.mach_absolute_time - mach_absolute_time.restype = ctypes.c_uint64 - - timebase = mach_timebase_info_data_t() - libc.mach_timebase_info(ctypes.byref(timebase)) - ticks_per_second = timebase.numer / timebase.denom * 1.0e9 - - def monotonic(): - """Monotonic clock, cannot go backward.""" - return mach_absolute_time() / ticks_per_second - - elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'): - if sys.platform.startswith('cygwin'): - # Note: cygwin implements clock_gettime (CLOCK_MONOTONIC = 4) since - # version 1.7.6. Using raw WinAPI for maximum version compatibility. - - # Ugly hack using the wrong calling convention (in 32-bit mode) - # because ctypes has no windll under cygwin (and it also seems that - # the code letting you select stdcall in _ctypes doesn't exist under - # the preprocessor definitions relevant to cygwin). - # This is 'safe' because: - # 1. The ABI of GetTickCount and GetTickCount64 is identical for - # both calling conventions because they both have no parameters. - # 2. libffi masks the problem because after making the call it doesn't - # touch anything through esp and epilogue code restores a correct - # esp from ebp afterwards. - try: - kernel32 = ctypes.cdll.kernel32 - except OSError: # 'No such file or directory' - kernel32 = ctypes.cdll.LoadLibrary('kernel32.dll') - else: - kernel32 = ctypes.windll.kernel32 - - GetTickCount64 = getattr(kernel32, 'GetTickCount64', None) - if GetTickCount64: - # Windows Vista / Windows Server 2008 or newer. - GetTickCount64.restype = ctypes.c_ulonglong - - def monotonic(): - """Monotonic clock, cannot go backward.""" - return GetTickCount64() / 1000.0 - - else: - # Before Windows Vista. - GetTickCount = kernel32.GetTickCount - GetTickCount.restype = ctypes.c_uint32 - - get_tick_count_lock = threading.Lock() - get_tick_count_last_sample = 0 - get_tick_count_wraparounds = 0 - - def monotonic(): - """Monotonic clock, cannot go backward.""" - global get_tick_count_last_sample - global get_tick_count_wraparounds - - with get_tick_count_lock: - current_sample = GetTickCount() - if current_sample < get_tick_count_last_sample: - get_tick_count_wraparounds += 1 - get_tick_count_last_sample = current_sample - - final_milliseconds = get_tick_count_wraparounds << 32 - final_milliseconds += get_tick_count_last_sample - return final_milliseconds / 1000.0 - - else: - try: - clock_gettime = ctypes.CDLL(ctypes.util.find_library('c'), - use_errno=True).clock_gettime - except Exception: - clock_gettime = ctypes.CDLL(ctypes.util.find_library('rt'), - use_errno=True).clock_gettime - - class timespec(ctypes.Structure): - """Time specification, as described in clock_gettime(3).""" - _fields_ = (('tv_sec', ctypes.c_long), - ('tv_nsec', ctypes.c_long)) - - if sys.platform.startswith('linux'): - CLOCK_MONOTONIC = 1 - elif sys.platform.startswith('freebsd'): - CLOCK_MONOTONIC = 4 - elif sys.platform.startswith('sunos5'): - CLOCK_MONOTONIC = 4 - elif 'bsd' in sys.platform: - CLOCK_MONOTONIC = 3 - elif sys.platform.startswith('aix'): - CLOCK_MONOTONIC = ctypes.c_longlong(10) - - def monotonic(): - """Monotonic clock, cannot go backward.""" - ts = timespec() - if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(ts)): - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return ts.tv_sec + ts.tv_nsec / 1.0e9 - - # Perform a sanity-check. - if monotonic() - monotonic() > 0: - raise ValueError('monotonic() is not monotonic!') - - except Exception as e: - raise RuntimeError('no suitable implementation for this system: ' + repr(e)) diff --git a/docs/configuration.rst b/docs/configuration.rst index 556f72d40a3..35bb63fac20 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -10,6 +10,12 @@ see specific integration documentation for more details. The following environment variables for the tracer are supported: +Common Configurations +--------------------- + +For common configuration variables (not language specific), see `Configure the Datadog Tracing Library`_. + + Unified Service Tagging ----------------------- @@ -177,18 +183,6 @@ Traces version_added: v2.17.0: - DD_TRACE_ENABLED: - type: Boolean - default: True - - description: | - Enable sending of spans to the Agent. Note that instrumentation will still be installed and spans will be - generated. - - version_added: - v0.41.0: | - Formerly named ``DATADOG_TRACE_ENABLED`` - DD_TRACE_HEADER_TAGS: description: | A map of case-insensitive http headers to tag names. Automatically applies matching header values as tags on request and response spans. For example if @@ -563,10 +557,10 @@ Test Visibility default: True description: | - Configures the ``CIVisibility`` service to query the Datadog API to decide whether to enable the Datadog - `Intelligent Test Runner _`. Setting the variable to - ``false`` will skip querying the API and disable code coverage - collection and test skipping. + Configures the ``CIVisibility`` service to query the Datadog API to decide whether to enable the Datadog `Test + Impact Analysis `_ (formerly Intelligent Test + Runner). Setting the variable to ``false`` will skip querying the API and disable code coverage collection and + test skipping. version_added: v1.13.0: @@ -597,6 +591,22 @@ Test Visibility version_added: v2.16.0: + DD_PYTEST_USE_NEW_PLUGIN_BETA: + type: Boolean + default: False + + description: | + Configures the ``CIVisibility`` service to use a beta release of the new version of the pytest plugin, + supporting `Auto Test Retries `_, + `Early Flake Detection `_, and + improved coverage collection for `Test Impact Analysis + `_. This version of the plugin will become the default in + the future. See the `release notes for v2.18.0 `_ + for more information. + + version_added: + v2.18.0: + DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS: type: Integer default: 500 @@ -882,6 +892,8 @@ Other .. _Unified Service Tagging: https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/ +.. _Configure the Datadog Tracing Library: https://docs.datadoghq.com/tracing/trace_collection/library_config/ + Profiling --------- diff --git a/docs/index.rst b/docs/index.rst index 2435da88217..3d19a4fbf14 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -52,6 +52,8 @@ contacting support. +--------------------------------------------------+---------------+----------------+ | :ref:`algoliasearch` | >= 2.5.0 | Yes | +--------------------------------------------------+---------------+----------------+ +| :ref:`anthropic` | >= 0.28.0 | Yes | ++--------------------------------------------------+---------------+----------------+ | :ref:`aredis` | \* | Yes | +--------------------------------------------------+---------------+----------------+ | :ref:`asgi` | >= 3.0 | No | @@ -172,6 +174,8 @@ contacting support. +--------------------------------------------------+---------------+----------------+ | :ref:`urllib3` | >= 1.25.8 | No | +--------------------------------------------------+---------------+----------------+ +| :ref:`vertexai` | >= 1.71.1 | Yes | ++--------------------------------------------------+---------------+----------------+ | :ref:`vertica` | >= 0.6 | Yes | +--------------------------------------------------+---------------+----------------+ | :ref:`wsgi` | \* | No | diff --git a/docs/integrations.rst b/docs/integrations.rst index 04a94007626..62e096aa668 100644 --- a/docs/integrations.rst +++ b/docs/integrations.rst @@ -55,6 +55,13 @@ aiohttp_jinja2 .. automodule:: ddtrace.contrib.aiohttp_jinja2 +.. _anthropic: + +anthropic +^^^^^^^^^^^^^^ +.. automodule:: ddtrace.contrib.anthropic + + .. _asyncio: asyncio @@ -481,7 +488,7 @@ urllib3 .. _vertexai: vertexai -^^^^^^^^^^^^^^^^^^^ +^^^^^^^^ .. automodule:: ddtrace.contrib.vertexai diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 6bfa3f90bc1..c9cc13a5a9e 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -9,6 +9,8 @@ aioredis algolia algoliasearch analytics +anthropic +Anthropic AnyCallable api app diff --git a/releasenotes/notes/iast-fix-awsgi-368c173e1f012400.yaml b/releasenotes/notes/iast-fix-awsgi-368c173e1f012400.yaml new file mode 100644 index 00000000000..4d40945744a --- /dev/null +++ b/releasenotes/notes/iast-fix-awsgi-368c173e1f012400.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + ASGI: This fix resolves an issue parsing response cookies in FastAPI and awsgi diff --git a/riotfile.py b/riotfile.py index e7a078a5425..69467452a1f 100644 --- a/riotfile.py +++ b/riotfile.py @@ -806,6 +806,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "django-q": latest, "spyne": latest, "zeep": latest, + "bcrypt": "==4.2.1", }, env={ "DD_CIVISIBILITY_ITR_ENABLED": "0", @@ -1609,7 +1610,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, env={ "DD_AGENT_PORT": "9126", - "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0", + "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1", }, venvs=[ Venv( @@ -1636,6 +1637,18 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "pytest": ["~=7.0", latest], "pytest-cov": "==2.12.0", }, + venvs=[ + Venv( + env={ + "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0", + }, + ), + Venv( + env={ + "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1", + }, + ), + ], ), ], ), @@ -1652,6 +1665,18 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "more_itertools": "<8.11.0", "httpx": latest, }, + venvs=[ + Venv( + env={ + "DD_PYTEST_USE_NEW_PLUGIN_BETA": "0", + }, + ), + Venv( + env={ + "DD_PYTEST_USE_NEW_PLUGIN_BETA": "1", + }, + ), + ], ), ], ), diff --git a/scripts/gen_circleci_config.py b/scripts/gen_circleci_config.py index 627a3715427..bc51f2c5519 100644 --- a/scripts/gen_circleci_config.py +++ b/scripts/gen_circleci_config.py @@ -17,9 +17,10 @@ def gen_required_suites(template: dict) -> None: required_suites = template["requires_tests"]["requires"] = [] for_each_testrun_needed( suites=sorted( - set(n for n, s in get_suites().items() if not s.get("skip", False)) & set(template["jobs"].keys()) + set(n.rpartition("::")[-1] for n, s in get_suites().items() if not s.get("skip", False)) + & set(template["jobs"].keys()) ), - action=lambda suite: required_suites.append(suite.rpartition("::")[-1]), + action=lambda suite: required_suites.append(suite), git_selections=extract_git_commit_selections(os.getenv("GIT_COMMIT_DESC", "")), ) diff --git a/scripts/iast/leak_functions.py b/scripts/iast/leak_functions.py index 55fdcb0bbaa..ebac2253858 100644 --- a/scripts/iast/leak_functions.py +++ b/scripts/iast/leak_functions.py @@ -13,7 +13,7 @@ from ddtrace.appsec._iast._iast_request_context import set_iast_request_enabled from ddtrace.appsec._iast._iast_request_context import start_iast_context from ddtrace.appsec._iast._taint_tracking import active_map_addreses_size -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from tests.utils import override_env diff --git a/scripts/iast/mod_leak_functions.py b/scripts/iast/mod_leak_functions.py index 40e7e5a99b7..bf96d93c497 100644 --- a/scripts/iast/mod_leak_functions.py +++ b/scripts/iast/mod_leak_functions.py @@ -13,8 +13,8 @@ import requests from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject v = SchemaValidator( diff --git a/scripts/iast/test_references.py b/scripts/iast/test_references.py index d4cdd2fcc16..8fce6e0cdd7 100644 --- a/scripts/iast/test_references.py +++ b/scripts/iast/test_references.py @@ -4,9 +4,9 @@ from mod_leak_functions import test_doit -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import reset_context +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted async def test_main(): diff --git a/supported_versions_output.json b/supported_versions_output.json new file mode 100644 index 00000000000..a51bb17bb9a --- /dev/null +++ b/supported_versions_output.json @@ -0,0 +1,309 @@ +[ + { + "integration": "aiobotocore", + "minimum_tracer_supported": "1.4.2", + "max_tracer_supported": "2.13.3", + "auto-instrumented": false + }, + { + "integration": "aiohttp", + "minimum_tracer_supported": "3.8.6", + "max_tracer_supported": "3.11.10", + "auto-instrumented": true + }, + { + "integration": "aiomysql", + "minimum_tracer_supported": "0.1.1", + "max_tracer_supported": "0.2.0", + "auto-instrumented": true + }, + { + "integration": "aiopg", + "minimum_tracer_supported": "1.4.0", + "max_tracer_supported": "1.4.0", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "algoliasearch", + "minimum_tracer_supported": "2.6.3", + "max_tracer_supported": "2.6.3", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "anthropic", + "minimum_tracer_supported": "0.26.0", + "max_tracer_supported": "0.40.0", + "auto-instrumented": true + }, + { + "integration": "aredis", + "minimum_tracer_supported": "1.1.8", + "max_tracer_supported": "1.1.8", + "auto-instrumented": true + }, + { + "integration": "asyncpg", + "minimum_tracer_supported": "0.23.0", + "max_tracer_supported": "0.30.0", + "auto-instrumented": true + }, + { + "integration": "avro", + "minimum_tracer_supported": "1.12.0", + "max_tracer_supported": "1.12.0", + "auto-instrumented": true + }, + { + "integration": "boto", + "minimum_tracer_supported": "2.49.0", + "max_tracer_supported": "2.49.0", + "auto-instrumented": true + }, + { + "integration": "botocore", + "minimum_tracer_supported": "1.20.106", + "max_tracer_supported": "1.35.78", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "bottle", + "minimum_tracer_supported": "0.12.25", + "max_tracer_supported": "0.13.2", + "auto-instrumented": true + }, + { + "integration": "celery", + "minimum_tracer_supported": "4.4.7", + "max_tracer_supported": "5.4.0", + "auto-instrumented": true + }, + { + "integration": "cherrypy", + "minimum_tracer_supported": "17.4.2", + "max_tracer_supported": "18.10.0", + "auto-instrumented": false + }, + { + "integration": "coverage", + "minimum_tracer_supported": "7.2.7", + "max_tracer_supported": "7.4.4", + "auto-instrumented": false + }, + { + "integration": "django", + "minimum_tracer_supported": "2.2.1", + "max_tracer_supported": "5.1", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "dramatiq", + "minimum_tracer_supported": "1.16.0", + "max_tracer_supported": "1.17.0", + "auto-instrumented": true + }, + { + "integration": "falcon", + "minimum_tracer_supported": "3.0.1", + "max_tracer_supported": "3.1.3", + "auto-instrumented": true + }, + { + "integration": "fastapi", + "minimum_tracer_supported": "0.64.0", + "max_tracer_supported": "0.115.6", + "auto-instrumented": true + }, + { + "integration": "flask", + "minimum_tracer_supported": "0.12.5", + "max_tracer_supported": "3.0.3", + "auto-instrumented": true + }, + { + "integration": "gevent", + "minimum_tracer_supported": "20.12.1", + "max_tracer_supported": "24.11.1", + "auto-instrumented": true + }, + { + "integration": "httpx", + "minimum_tracer_supported": "0.15.4", + "max_tracer_supported": "0.28.1", + "auto-instrumented": true + }, + { + "integration": "jinja2", + "minimum_tracer_supported": "2.10.3", + "max_tracer_supported": "3.1.4", + "auto-instrumented": true + }, + { + "integration": "kombu", + "minimum_tracer_supported": "4.2.2.post1", + "max_tracer_supported": "5.4.2", + "auto-instrumented": false + }, + { + "integration": "langchain", + "minimum_tracer_supported": "0.0.192", + "max_tracer_supported": "0.3.10", + "auto-instrumented": true + }, + { + "integration": "logbook", + "minimum_tracer_supported": "1.0.0", + "max_tracer_supported": "1.7.0.post0", + "auto-instrumented": false + }, + { + "integration": "loguru", + "minimum_tracer_supported": "0.4.1", + "max_tracer_supported": "0.7.3", + "auto-instrumented": false + }, + { + "integration": "mako", + "minimum_tracer_supported": "1.1.6", + "max_tracer_supported": "1.3.5", + "auto-instrumented": true + }, + { + "integration": "mariadb", + "minimum_tracer_supported": "1.0.11", + "max_tracer_supported": "1.1.11", + "auto-instrumented": true + }, + { + "integration": "molten", + "minimum_tracer_supported": "1.0.2", + "max_tracer_supported": "1.0.2", + "auto-instrumented": true + }, + { + "integration": "mongoengine", + "minimum_tracer_supported": "0.29.1", + "max_tracer_supported": "0.29.1", + "auto-instrumented": true + }, + { + "integration": "openai", + "minimum_tracer_supported": "0.26.5", + "max_tracer_supported": "1.57.2", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "protobuf", + "minimum_tracer_supported": "3.8.0", + "max_tracer_supported": "5.28.3", + "auto-instrumented": false + }, + { + "integration": "pylibmc", + "minimum_tracer_supported": "1.6.3", + "max_tracer_supported": "1.6.3", + "auto-instrumented": true + }, + { + "integration": "pymemcache", + "minimum_tracer_supported": "3.4.4", + "max_tracer_supported": "4.0.0", + "auto-instrumented": true + }, + { + "integration": "pymongo", + "minimum_tracer_supported": "3.8.0", + "max_tracer_supported": "4.10.1", + "auto-instrumented": true + }, + { + "integration": "pymysql", + "minimum_tracer_supported": "0.10.1", + "max_tracer_supported": "1.1.1", + "auto-instrumented": true + }, + { + "integration": "pynamodb", + "minimum_tracer_supported": "5.5.1", + "max_tracer_supported": "5.5.1", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "pyodbc", + "minimum_tracer_supported": "4.0.39", + "max_tracer_supported": "5.2.0", + "auto-instrumented": true + }, + { + "integration": "pyramid", + "minimum_tracer_supported": "1.10.8", + "max_tracer_supported": "2.0.2", + "auto-instrumented": true + }, + { + "integration": "redis", + "minimum_tracer_supported": "2.10.6", + "max_tracer_supported": "5.2.1", + "auto-instrumented": true + }, + { + "integration": "requests", + "minimum_tracer_supported": "2.20.1", + "max_tracer_supported": "2.32.3", + "auto-instrumented": true + }, + { + "integration": "sanic", + "minimum_tracer_supported": "20.12.7", + "max_tracer_supported": "24.6.0", + "auto-instrumented": true + }, + { + "integration": "sqlalchemy", + "minimum_tracer_supported": "1.2.19", + "max_tracer_supported": "2.0.36", + "auto-instrumented": false + }, + { + "integration": "starlette", + "minimum_tracer_supported": "0.13.6", + "max_tracer_supported": "0.41.3", + "auto-instrumented": true + }, + { + "integration": "structlog", + "minimum_tracer_supported": "20.2.0", + "max_tracer_supported": "24.4.0", + "auto-instrumented": false + }, + { + "integration": "tornado", + "minimum_tracer_supported": "4.5.3", + "max_tracer_supported": "6.4", + "pinned": "true", + "auto-instrumented": false + }, + { + "integration": "urllib3", + "minimum_tracer_supported": "1.24.3", + "max_tracer_supported": "2.2.3", + "auto-instrumented": false + }, + { + "integration": "vertexai", + "minimum_tracer_supported": "1.71.1", + "max_tracer_supported": "1.71.1", + "auto-instrumented": true + }, + { + "integration": "yaaredis", + "minimum_tracer_supported": "2.0.4", + "max_tracer_supported": "3.0.0", + "auto-instrumented": true + } +] \ No newline at end of file diff --git a/supported_versions_table.csv b/supported_versions_table.csv new file mode 100644 index 00000000000..3f7384a0cdd --- /dev/null +++ b/supported_versions_table.csv @@ -0,0 +1,51 @@ +integration,minimum_tracer_supported,max_tracer_supported,auto-instrumented +aiobotocore,1.4.2,2.13.3,False +aiohttp,3.8.6,3.11.10,True +aiomysql,0.1.1,0.2.0,True +aiopg *,1.4.0,1.4.0,True +algoliasearch *,2.6.3,2.6.3,True +anthropic,0.26.0,0.40.0,True +aredis,1.1.8,1.1.8,True +asyncpg,0.23.0,0.30.0,True +avro,1.12.0,1.12.0,True +boto,2.49.0,2.49.0,True +botocore *,1.20.106,1.35.78,True +bottle,0.12.25,0.13.2,True +celery,4.4.7,5.4.0,True +cherrypy,17.4.2,18.10.0,False +coverage,7.2.7,7.4.4,False +django *,2.2.1,5.1,True +dramatiq,1.16.0,1.17.0,True +falcon,3.0.1,3.1.3,True +fastapi,0.64.0,0.115.6,True +flask,0.12.5,3.0.3,True +gevent,20.12.1,24.11.1,True +httpx,0.15.4,0.28.1,True +jinja2,2.10.3,3.1.4,True +kombu,4.2.2.post1,5.4.2,False +langchain,0.0.192,0.3.10,True +logbook,1.0.0,1.7.0.post0,False +loguru,0.4.1,0.7.3,False +mako,1.1.6,1.3.5,True +mariadb,1.0.11,1.1.11,True +molten,1.0.2,1.0.2,True +mongoengine,0.29.1,0.29.1,True +openai *,0.26.5,1.57.2,True +protobuf,3.8.0,5.28.3,False +pylibmc,1.6.3,1.6.3,True +pymemcache,3.4.4,4.0.0,True +pymongo,3.8.0,4.10.1,True +pymysql,0.10.1,1.1.1,True +pynamodb *,5.5.1,5.5.1,True +pyodbc,4.0.39,5.2.0,True +pyramid,1.10.8,2.0.2,True +redis,2.10.6,5.2.1,True +requests,2.20.1,2.32.3,True +sanic,20.12.7,24.6.0,True +sqlalchemy,1.2.19,2.0.36,False +starlette,0.13.6,0.41.3,True +structlog,20.2.0,24.4.0,False +tornado *,4.5.3,6.4,False +urllib3,1.24.3,2.2.3,False +vertexai,1.71.1,1.71.1,True +yaaredis,2.0.4,3.0.0,True diff --git a/tests/appsec/app.py b/tests/appsec/app.py index 103341c752a..eb5beb666cf 100644 --- a/tests/appsec/app.py +++ b/tests/appsec/app.py @@ -239,7 +239,7 @@ def iast_ast_patching_io_bytes_io_untainted(): changed = BytesIO(bytes_filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not is_pyobject_tainted(changed): resp = Response("OK") @@ -270,7 +270,7 @@ def iast_ast_patching_io_string_io_untainted(): changed = StringIO(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not is_pyobject_tainted(changed): resp = Response("OK") @@ -302,7 +302,7 @@ def iast_ast_patching_io_bytes_io_read_untainted(): changed = BytesIO(bytes_filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not is_pyobject_tainted(changed.read(4)): resp = Response("OK") @@ -333,7 +333,7 @@ def iast_ast_patching_io_string_io_read_untainted(): changed = StringIO(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not is_pyobject_tainted(changed.read(4)): resp = Response("OK") @@ -365,7 +365,7 @@ def iast_ast_patching_io_bytes_io(): changed = BytesIO(bytes_filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed): resp = Response("OK") @@ -396,7 +396,7 @@ def iast_ast_patching_io_string_io(): changed = StringIO(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed): resp = Response("OK") @@ -428,7 +428,7 @@ def iast_ast_patching_io_bytes_io_read(): changed = BytesIO(bytes_filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed.read(4)): resp = Response("OK") @@ -459,7 +459,7 @@ def iast_ast_patching_io_string_io_read(): changed = StringIO(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed.read(4)): resp = Response("OK") @@ -479,7 +479,7 @@ def iast_ast_patching_re_sub(): changed = pattern.sub(" ", filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed): resp = Response("OK") @@ -501,7 +501,7 @@ def iast_ast_patching_non_re_sub(): changed = pattern.sub(" ", filename) resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed): resp = Response("Fail") @@ -521,7 +521,7 @@ def iast_ast_patching_re_subn(): changed, number = pattern.subn(" ", filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed): resp = Response("OK") @@ -543,7 +543,7 @@ def iast_ast_patching_non_re_subn(): changed, number = pattern.subn(" ", filename) resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if is_pyobject_tainted(changed): resp = Response("Fail") @@ -563,7 +563,7 @@ def iast_ast_patching_re_split(): result = pattern.split(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if all(map(is_pyobject_tainted, result)): resp = Response("OK") @@ -585,7 +585,7 @@ def iast_ast_patching_non_re_split(): result = pattern.split(filename) resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if any(map(is_pyobject_tainted, result)): resp = Response("Fail") @@ -605,7 +605,7 @@ def iast_ast_patching_re_findall(): result = pattern.findall(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if all(map(is_pyobject_tainted, result)): resp = Response("OK") @@ -627,7 +627,7 @@ def iast_ast_patching_non_re_findall(): result = pattern.findall(filename) resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if any(map(is_pyobject_tainted, result)): resp = Response("Fail") @@ -647,7 +647,7 @@ def iast_ast_patching_re_finditer(): result = pattern.finditer(filename) resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if all(map(is_pyobject_tainted, result)): resp = Response("OK") @@ -669,7 +669,7 @@ def iast_ast_patching_non_re_finditer(): result = pattern.finditer(filename) resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if any(map(is_pyobject_tainted, result)): resp = Response("Fail") @@ -697,7 +697,7 @@ def iast_ast_patching_re_groups(): result = [] resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if result and all(map(is_pyobject_tainted, result)): resp = Response("OK") @@ -727,7 +727,7 @@ def iast_ast_patching_non_re_groups(): result = [] resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not result or any(map(is_pyobject_tainted, result)): resp = Response("Fail") @@ -755,7 +755,7 @@ def iast_ast_patching_re_string(): result = None resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if result and is_pyobject_tainted(result): resp = Response("OK") @@ -785,7 +785,7 @@ def iast_ast_patching_non_re_string(): result = None resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not result or is_pyobject_tainted(result): resp = Response("Fail") @@ -813,7 +813,7 @@ def iast_ast_patching_re_fullmatch(): result = [] resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if result and all(map(is_pyobject_tainted, result)): resp = Response("OK") @@ -843,7 +843,7 @@ def iast_ast_patching_non_re_fullmatch(): result = [] resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not result or any(map(is_pyobject_tainted, result)): resp = Response("Fail") @@ -871,7 +871,7 @@ def iast_ast_patching_re_expand(): result = None resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if result and is_pyobject_tainted(result): resp = Response("OK") @@ -901,7 +901,7 @@ def iast_ast_patching_non_re_expand(): result = None resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not result or is_pyobject_tainted(result): resp = Response("Fail") @@ -929,7 +929,7 @@ def iast_ast_patching_re_search(): result = [] resp = Response("Fail") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if result and all(map(is_pyobject_tainted, result)): resp = Response("OK") @@ -959,7 +959,7 @@ def iast_ast_patching_non_re_search(): result = [] resp = Response("OK") try: - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted if not result or any(map(is_pyobject_tainted, result)): resp = Response("Fail") diff --git a/tests/appsec/iast/aspects/aspect_utils.py b/tests/appsec/iast/aspects/aspect_utils.py index e62625e256b..0467915e672 100644 --- a/tests/appsec/iast/aspects/aspect_utils.py +++ b/tests/appsec/iast/aspects/aspect_utils.py @@ -11,7 +11,7 @@ from ddtrace.appsec._iast._taint_tracking import TaintRange from ddtrace.appsec._iast._taint_tracking import as_formatted_evidence from ddtrace.appsec._iast._taint_tracking import set_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject_with_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject_with_ranges from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_add_aspect.py b/tests/appsec/iast/aspects/test_add_aspect.py index f9f86a4413c..a2e4558198f 100644 --- a/tests/appsec/iast/aspects/test_add_aspect.py +++ b/tests/appsec/iast/aspects/test_add_aspect.py @@ -5,12 +5,12 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import TaintRange_ +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from tests.appsec.iast.conftest import _end_iast_context_and_oce diff --git a/tests/appsec/iast/aspects/test_add_aspect_fixtures.py b/tests/appsec/iast/aspects/test_add_aspect_fixtures.py index 19a6a97dae7..854cbea2032 100644 --- a/tests/appsec/iast/aspects/test_add_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_add_aspect_fixtures.py @@ -4,8 +4,8 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_add_inplace_aspect.py b/tests/appsec/iast/aspects/test_add_inplace_aspect.py index b6d2b22dce8..babe9bae651 100644 --- a/tests/appsec/iast/aspects/test_add_inplace_aspect.py +++ b/tests/appsec/iast/aspects/test_add_inplace_aspect.py @@ -5,10 +5,10 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import TaintRange_ +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects diff --git a/tests/appsec/iast/aspects/test_add_inplace_aspect_fixtures.py b/tests/appsec/iast/aspects/test_add_inplace_aspect_fixtures.py index 1d59ba41dbc..4fc96486be0 100644 --- a/tests/appsec/iast/aspects/test_add_inplace_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_add_inplace_aspect_fixtures.py @@ -4,8 +4,8 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_asyncio.py b/tests/appsec/iast/aspects/test_asyncio.py index e01627819b3..1bd8bd7b45a 100644 --- a/tests/appsec/iast/aspects/test_asyncio.py +++ b/tests/appsec/iast/aspects/test_asyncio.py @@ -5,9 +5,9 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_bytearray_extend_aspect.py b/tests/appsec/iast/aspects/test_bytearray_extend_aspect.py index e746800cd3f..41047e47203 100644 --- a/tests/appsec/iast/aspects/test_bytearray_extend_aspect.py +++ b/tests/appsec/iast/aspects/test_bytearray_extend_aspect.py @@ -6,10 +6,10 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.utils import override_global_config diff --git a/tests/appsec/iast/aspects/test_common_replace_aspects.py b/tests/appsec/iast/aspects/test_common_replace_aspects.py index f858c065aa1..7054b34e83b 100644 --- a/tests/appsec/iast/aspects/test_common_replace_aspects.py +++ b/tests/appsec/iast/aspects/test_common_replace_aspects.py @@ -3,8 +3,8 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_encode_decode_aspect.py b/tests/appsec/iast/aspects/test_encode_decode_aspect.py index 6e994f6e9fa..a5da649c1fe 100644 --- a/tests/appsec/iast/aspects/test_encode_decode_aspect.py +++ b/tests/appsec/iast/aspects/test_encode_decode_aspect.py @@ -3,8 +3,8 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_format_aspect_fixtures.py b/tests/appsec/iast/aspects/test_format_aspect_fixtures.py index a35a424d67b..b401589af92 100644 --- a/tests/appsec/iast/aspects/test_format_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_format_aspect_fixtures.py @@ -9,10 +9,10 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import as_formatted_evidence -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.aspect_utils import BaseReplacement from tests.appsec.iast.aspects.aspect_utils import create_taint_range_with_format from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_index_aspect_fixtures.py b/tests/appsec/iast/aspects/test_index_aspect_fixtures.py index 0542cd636c8..8ab46d9bf45 100644 --- a/tests/appsec/iast/aspects/test_index_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_index_aspect_fixtures.py @@ -4,10 +4,10 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.utils import override_global_config diff --git a/tests/appsec/iast/aspects/test_io_aspects.py b/tests/appsec/iast/aspects/test_io_aspects.py index ea74825895d..177e0742674 100644 --- a/tests/appsec/iast/aspects/test_io_aspects.py +++ b/tests/appsec/iast/aspects/test_io_aspects.py @@ -3,9 +3,9 @@ from ddtrace.appsec._common_module_patches import patch_common_modules from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast._taint_tracking.aspects import bytesio_aspect from ddtrace.appsec._iast._taint_tracking.aspects import stringio_aspect diff --git a/tests/appsec/iast/aspects/test_join_aspect_fixtures.py b/tests/appsec/iast/aspects/test_join_aspect_fixtures.py index 8692485f295..d4c896786f6 100644 --- a/tests/appsec/iast/aspects/test_join_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_join_aspect_fixtures.py @@ -5,10 +5,10 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.utils import override_global_config diff --git a/tests/appsec/iast/aspects/test_modulo_aspect_fixtures.py b/tests/appsec/iast/aspects/test_modulo_aspect_fixtures.py index 80ca12a2db8..175b5bf9439 100644 --- a/tests/appsec/iast/aspects/test_modulo_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_modulo_aspect_fixtures.py @@ -11,8 +11,8 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import as_formatted_evidence from ddtrace.appsec._iast._taint_tracking import get_ranges -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.aspect_utils import BaseReplacement from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_ospath_aspects.py b/tests/appsec/iast/aspects/test_ospath_aspects.py index 976327cdd2c..9e1b5eee93f 100644 --- a/tests/appsec/iast/aspects/test_ospath_aspects.py +++ b/tests/appsec/iast/aspects/test_ospath_aspects.py @@ -18,8 +18,8 @@ from ddtrace.appsec._iast._taint_tracking.aspects import ospathsplitdrive_aspect if sys.version_info >= (3, 12): from ddtrace.appsec._iast._taint_tracking.aspects import ospathsplitroot_aspect -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject def test_ospathjoin_first_arg_nottainted_noslash(): diff --git a/tests/appsec/iast/aspects/test_ospath_aspects_fixtures.py b/tests/appsec/iast/aspects/test_ospath_aspects_fixtures.py index 68ac2cba76e..7cd2069dbfd 100644 --- a/tests/appsec/iast/aspects/test_ospath_aspects_fixtures.py +++ b/tests/appsec/iast/aspects/test_ospath_aspects_fixtures.py @@ -8,10 +8,10 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.utils import override_global_config diff --git a/tests/appsec/iast/aspects/test_other_patching.py b/tests/appsec/iast/aspects/test_other_patching.py index d392fdb7c4b..a7b620a1792 100644 --- a/tests/appsec/iast/aspects/test_other_patching.py +++ b/tests/appsec/iast/aspects/test_other_patching.py @@ -5,8 +5,8 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/aspects/test_re_aspects.py b/tests/appsec/iast/aspects/test_re_aspects.py index b5069948a89..55d2ea81f03 100644 --- a/tests/appsec/iast/aspects/test_re_aspects.py +++ b/tests/appsec/iast/aspects/test_re_aspects.py @@ -6,9 +6,9 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast._taint_tracking.aspects import index_aspect from ddtrace.appsec._iast._taint_tracking.aspects import re_expand_aspect diff --git a/tests/appsec/iast/aspects/test_replace_aspect.py b/tests/appsec/iast/aspects/test_replace_aspect.py index b30fa7cdede..b929817e3eb 100644 --- a/tests/appsec/iast/aspects/test_replace_aspect.py +++ b/tests/appsec/iast/aspects/test_replace_aspect.py @@ -7,9 +7,9 @@ from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange from ddtrace.appsec._iast._taint_tracking import as_formatted_evidence -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking import set_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects from ddtrace.internal.compat import PYTHON_VERSION_INFO diff --git a/tests/appsec/iast/aspects/test_side_effects.py b/tests/appsec/iast/aspects/test_side_effects.py index 0c019f9994b..120b8f88a05 100644 --- a/tests/appsec/iast/aspects/test_side_effects.py +++ b/tests/appsec/iast/aspects/test_side_effects.py @@ -3,9 +3,9 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject -from ddtrace.appsec._iast._taint_tracking import taint_pyobject_with_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject_with_ranges import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.appsec.iast.iast_utils_side_effects import MagicMethodsException diff --git a/tests/appsec/iast/aspects/test_slice_aspect_fixtures.py b/tests/appsec/iast/aspects/test_slice_aspect_fixtures.py index bd42b136e06..6869fbbd15c 100644 --- a/tests/appsec/iast/aspects/test_slice_aspect_fixtures.py +++ b/tests/appsec/iast/aspects/test_slice_aspect_fixtures.py @@ -5,10 +5,10 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.utils import override_global_config diff --git a/tests/appsec/iast/aspects/test_split_aspect.py b/tests/appsec/iast/aspects/test_split_aspect.py index 30f4fe121ca..faee670d12e 100644 --- a/tests/appsec/iast/aspects/test_split_aspect.py +++ b/tests/appsec/iast/aspects/test_split_aspect.py @@ -9,11 +9,11 @@ from ddtrace.appsec._iast._taint_tracking import _aspect_rsplit from ddtrace.appsec._iast._taint_tracking import _aspect_split from ddtrace.appsec._iast._taint_tracking import _aspect_splitlines -from ddtrace.appsec._iast._taint_tracking import create_context from ddtrace.appsec._iast._taint_tracking import get_ranges -from ddtrace.appsec._iast._taint_tracking import reset_context from ddtrace.appsec._iast._taint_tracking import set_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.test_aspect_helpers import _build_sample_range from tests.utils import override_global_config diff --git a/tests/appsec/iast/aspects/test_str_aspect.py b/tests/appsec/iast/aspects/test_str_aspect.py index ba32fa970b5..c036a5fbbe3 100644 --- a/tests/appsec/iast/aspects/test_str_aspect.py +++ b/tests/appsec/iast/aspects/test_str_aspect.py @@ -6,9 +6,9 @@ from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange from ddtrace.appsec._iast._taint_tracking import as_formatted_evidence -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects from tests.appsec.iast.aspects.aspect_utils import BaseReplacement from tests.appsec.iast.aspects.aspect_utils import create_taint_range_with_format diff --git a/tests/appsec/iast/fixtures/entrypoint/views.py b/tests/appsec/iast/fixtures/entrypoint/views.py index 58baf906c53..3359e5c0366 100644 --- a/tests/appsec/iast/fixtures/entrypoint/views.py +++ b/tests/appsec/iast/fixtures/entrypoint/views.py @@ -3,9 +3,9 @@ def add_test(): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import create_context - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._context import create_context + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject string_to_taint = "abc" create_context() diff --git a/tests/appsec/iast/fixtures/taint_sinks/sql_injection_psycopg2.py b/tests/appsec/iast/fixtures/taint_sinks/sql_injection_psycopg2.py index 3411509c956..69994a20fe1 100644 --- a/tests/appsec/iast/fixtures/taint_sinks/sql_injection_psycopg2.py +++ b/tests/appsec/iast/fixtures/taint_sinks/sql_injection_psycopg2.py @@ -3,8 +3,8 @@ import psycopg2 from psycopg2.errors import DuplicateTable -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted POSTGRES_HOST = os.getenv("TEST_POSTGRES_HOST", "127.0.0.1") diff --git a/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlalchemy.py b/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlalchemy.py index f8910b6b5cd..29b014ee364 100644 --- a/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlalchemy.py +++ b/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlalchemy.py @@ -2,8 +2,8 @@ from sqlalchemy import text from sqlalchemy.exc import ProgrammingError -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted def sqli_simple(table): diff --git a/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlite3.py b/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlite3.py index 08d98abf24d..3d8ee69f8b3 100644 --- a/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlite3.py +++ b/tests/appsec/iast/fixtures/taint_sinks/sql_injection_sqlite3.py @@ -1,7 +1,7 @@ import sqlite3 -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted def sqli_simple(table): diff --git a/tests/appsec/iast/taint_sinks/test_command_injection.py b/tests/appsec/iast/taint_sinks/test_command_injection.py index a18fac45de1..b716f594e85 100644 --- a/tests/appsec/iast/taint_sinks/test_command_injection.py +++ b/tests/appsec/iast/taint_sinks/test_command_injection.py @@ -7,8 +7,8 @@ from ddtrace.appsec._iast._iast_request_context import get_iast_reporter from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_CMDI from ddtrace.appsec._iast.taint_sinks.command_injection import patch diff --git a/tests/appsec/iast/taint_sinks/test_command_injection_redacted.py b/tests/appsec/iast/taint_sinks/test_command_injection_redacted.py index f1e2b98089c..4c25cda8dc2 100644 --- a/tests/appsec/iast/taint_sinks/test_command_injection_redacted.py +++ b/tests/appsec/iast/taint_sinks/test_command_injection_redacted.py @@ -3,7 +3,7 @@ from ddtrace.appsec._iast._taint_tracking import origin_to_str from ddtrace.appsec._iast._taint_tracking import str_to_origin -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_CMDI from ddtrace.appsec._iast.reporter import Evidence diff --git a/tests/appsec/iast/taint_sinks/test_header_injection_redacted.py b/tests/appsec/iast/taint_sinks/test_header_injection_redacted.py index d47433f7745..61a3aa83a49 100644 --- a/tests/appsec/iast/taint_sinks/test_header_injection_redacted.py +++ b/tests/appsec/iast/taint_sinks/test_header_injection_redacted.py @@ -2,10 +2,10 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking import origin_to_str from ddtrace.appsec._iast._taint_tracking import str_to_origin -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_HEADER_INJECTION from ddtrace.appsec._iast.reporter import Evidence diff --git a/tests/appsec/iast/taint_sinks/test_path_traversal.py b/tests/appsec/iast/taint_sinks/test_path_traversal.py index b195edc2427..cc016eb29fb 100644 --- a/tests/appsec/iast/taint_sinks/test_path_traversal.py +++ b/tests/appsec/iast/taint_sinks/test_path_traversal.py @@ -4,7 +4,7 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import DEFAULT_PATH_TRAVERSAL_FUNCTIONS from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/taint_sinks/test_path_traversal_redacted.py b/tests/appsec/iast/taint_sinks/test_path_traversal_redacted.py index 181af423c9c..996bc2ee356 100644 --- a/tests/appsec/iast/taint_sinks/test_path_traversal_redacted.py +++ b/tests/appsec/iast/taint_sinks/test_path_traversal_redacted.py @@ -4,7 +4,7 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL from ddtrace.appsec._iast.reporter import Evidence from ddtrace.appsec._iast.reporter import IastSpanReporter diff --git a/tests/appsec/iast/taint_sinks/test_sql_injection.py b/tests/appsec/iast/taint_sinks/test_sql_injection.py index d8fe767efb6..bf2190cdf99 100644 --- a/tests/appsec/iast/taint_sinks/test_sql_injection.py +++ b/tests/appsec/iast/taint_sinks/test_sql_injection.py @@ -1,8 +1,8 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.appsec._iast.taint_sinks._base import VulnerabilityBase from tests.appsec.iast.aspects.conftest import _iast_patched_module diff --git a/tests/appsec/iast/taint_sinks/test_sql_injection_redacted.py b/tests/appsec/iast/taint_sinks/test_sql_injection_redacted.py index ba6675e7531..01645cf1d39 100644 --- a/tests/appsec/iast/taint_sinks/test_sql_injection_redacted.py +++ b/tests/appsec/iast/taint_sinks/test_sql_injection_redacted.py @@ -1,10 +1,10 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking import origin_to_str from ddtrace.appsec._iast._taint_tracking import str_to_origin -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.appsec._iast.reporter import Evidence diff --git a/tests/appsec/iast/taint_sinks/test_ssrf.py b/tests/appsec/iast/taint_sinks/test_ssrf.py index 8b35013b873..f6f3ea0fb58 100644 --- a/tests/appsec/iast/taint_sinks/test_ssrf.py +++ b/tests/appsec/iast/taint_sinks/test_ssrf.py @@ -1,5 +1,5 @@ from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.contrib.httplib.patch import patch as httplib_patch diff --git a/tests/appsec/iast/taint_sinks/test_ssrf_redacted.py b/tests/appsec/iast/taint_sinks/test_ssrf_redacted.py index aa316ab3b02..d5f60e8878e 100644 --- a/tests/appsec/iast/taint_sinks/test_ssrf_redacted.py +++ b/tests/appsec/iast/taint_sinks/test_ssrf_redacted.py @@ -4,7 +4,7 @@ from ddtrace.appsec._iast._taint_tracking import origin_to_str from ddtrace.appsec._iast._taint_tracking import str_to_origin -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_SSRF from ddtrace.appsec._iast.reporter import Evidence diff --git a/tests/appsec/iast/taint_tracking/test_native_taint_range.py b/tests/appsec/iast/taint_tracking/test_native_taint_range.py index 00079d7772b..d1b862b73a3 100644 --- a/tests/appsec/iast/taint_tracking/test_native_taint_range.py +++ b/tests/appsec/iast/taint_tracking/test_native_taint_range.py @@ -14,19 +14,19 @@ from ddtrace.appsec._iast._taint_tracking import Source from ddtrace.appsec._iast._taint_tracking import TaintRange from ddtrace.appsec._iast._taint_tracking import are_all_text_all_ranges -from ddtrace.appsec._iast._taint_tracking import create_context from ddtrace.appsec._iast._taint_tracking import debug_taint_map from ddtrace.appsec._iast._taint_tracking import get_range_by_hash from ddtrace.appsec._iast._taint_tracking import get_ranges -from ddtrace.appsec._iast._taint_tracking import is_notinterned_notfasttainted_unicode from ddtrace.appsec._iast._taint_tracking import num_objects_tainted -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import reset_contexts -from ddtrace.appsec._iast._taint_tracking import set_fast_tainted_if_notinterned_unicode from ddtrace.appsec._iast._taint_tracking import set_ranges from ddtrace.appsec._iast._taint_tracking import shift_taint_range from ddtrace.appsec._iast._taint_tracking import shift_taint_ranges -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._context import reset_contexts +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import is_notinterned_notfasttainted_unicode +from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import set_fast_tainted_if_notinterned_unicode +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast._taint_tracking.aspects import bytearray_extend_aspect as extend_aspect from ddtrace.appsec._iast._taint_tracking.aspects import format_aspect diff --git a/tests/appsec/iast/taint_tracking/test_taint_tracking.py b/tests/appsec/iast/taint_tracking/test_taint_tracking.py index ac3d009633f..0844b24fbd8 100644 --- a/tests/appsec/iast/taint_tracking/test_taint_tracking.py +++ b/tests/appsec/iast/taint_tracking/test_taint_tracking.py @@ -13,9 +13,9 @@ from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import TaintRange from ddtrace.appsec._iast._taint_tracking import num_objects_tainted - from ddtrace.appsec._iast._taint_tracking import reset_context from ddtrace.appsec._iast._taint_tracking import set_ranges - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._context import reset_context + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect diff --git a/tests/appsec/iast/test_env_var.py b/tests/appsec/iast/test_env_var.py index 57604815aac..18fe79229ed 100644 --- a/tests/appsec/iast/test_env_var.py +++ b/tests/appsec/iast/test_env_var.py @@ -127,13 +127,6 @@ def test_env_var_iast_disabled_parametrized(capfd, configuration_endpoint, env_v assert "IAST enabled" not in captured.err -@pytest.mark.subprocess( - env=dict(DD_IAST_ENABLED="False"), err=b"WARNING:root:IAST not enabled but native module is being loaded\n" -) -def test_env_var_iast_disabled_native_module_warning(): - import ddtrace.appsec._iast._taint_tracking._native # noqa: F401 - - @pytest.mark.subprocess(env=dict(DD_IAST_ENABLED="True"), err=None) def test_env_var_iast_enabled_no__native_module_warning(): import ddtrace.appsec._iast._taint_tracking._native # noqa: F401 diff --git a/tests/appsec/iast/test_grpc_iast.py b/tests/appsec/iast/test_grpc_iast.py index 47104e0915e..ba7b6027759 100644 --- a/tests/appsec/iast/test_grpc_iast.py +++ b/tests/appsec/iast/test_grpc_iast.py @@ -28,7 +28,7 @@ def iast_c_context(): def _check_test_range(value): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges = get_tainted_ranges(value) assert len(ranges) == 1, f"found {len(ranges)} ranges" diff --git a/tests/appsec/iast/test_iast_propagation_path.py b/tests/appsec/iast/test_iast_propagation_path.py index c9c32b7258e..229e3abbc55 100644 --- a/tests/appsec/iast/test_iast_propagation_path.py +++ b/tests/appsec/iast/test_iast_propagation_path.py @@ -2,7 +2,7 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.appsec.iast.iast_utils import get_line_and_hash diff --git a/tests/appsec/iast/test_json_tainting.py b/tests/appsec/iast/test_json_tainting.py index 2678fd70487..43c9370f306 100644 --- a/tests/appsec/iast/test_json_tainting.py +++ b/tests/appsec/iast/test_json_tainting.py @@ -3,9 +3,9 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_utils import LazyTaintDict from ddtrace.appsec._iast._taint_utils import LazyTaintList from tests.utils import override_global_config diff --git a/tests/appsec/iast/test_taint_utils.py b/tests/appsec/iast/test_taint_utils.py index 6749c2788ec..9e32b0e4049 100644 --- a/tests/appsec/iast/test_taint_utils.py +++ b/tests/appsec/iast/test_taint_utils.py @@ -2,8 +2,8 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_utils import LazyTaintDict from ddtrace.appsec._iast._taint_utils import LazyTaintList from ddtrace.appsec._iast._taint_utils import check_tainted_dbapi_args diff --git a/tests/appsec/iast/test_telemetry.py b/tests/appsec/iast/test_telemetry.py index 95b9b8aeb45..dc07754bdc5 100644 --- a/tests/appsec/iast/test_telemetry.py +++ b/tests/appsec/iast/test_telemetry.py @@ -14,7 +14,7 @@ from ddtrace.appsec._iast._patch_modules import patch_iast from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import origin_to_str -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import VULN_CMDI from ddtrace.appsec._iast.constants import VULN_HEADER_INJECTION from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL diff --git a/tests/appsec/iast_memcheck/test_iast_mem_check.py b/tests/appsec/iast_memcheck/test_iast_mem_check.py index d427f124aae..c049dd3c572 100644 --- a/tests/appsec/iast_memcheck/test_iast_mem_check.py +++ b/tests/appsec/iast_memcheck/test_iast_mem_check.py @@ -7,12 +7,12 @@ from ddtrace.appsec._iast._stacktrace import get_info_frame from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking import active_map_addreses_size -from ddtrace.appsec._iast._taint_tracking import create_context -from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import initializer_size from ddtrace.appsec._iast._taint_tracking import num_objects_tainted -from ddtrace.appsec._iast._taint_tracking import reset_context -from ddtrace.appsec._iast._taint_tracking import taint_pyobject +from ddtrace.appsec._iast._taint_tracking._context import create_context +from ddtrace.appsec._iast._taint_tracking._context import reset_context +from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.appsec.iast.taint_sinks.conftest import _get_span_report from tests.appsec.iast_memcheck._stacktrace_py import get_info_frame as get_info_frame_py diff --git a/tests/appsec/iast_packages/packages/pkg_attrs.py b/tests/appsec/iast_packages/packages/pkg_attrs.py index 2d32ce4b7a2..fb38d8e3237 100644 --- a/tests/appsec/iast_packages/packages/pkg_attrs.py +++ b/tests/appsec/iast_packages/packages/pkg_attrs.py @@ -37,7 +37,7 @@ class User: def pkg_attrs_propagation_view(): import attrs - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_beautifulsoup4.py b/tests/appsec/iast_packages/packages/pkg_beautifulsoup4.py index b6c55056165..d99f5f63eb2 100644 --- a/tests/appsec/iast_packages/packages/pkg_beautifulsoup4.py +++ b/tests/appsec/iast_packages/packages/pkg_beautifulsoup4.py @@ -30,7 +30,7 @@ def pkg_beautifulsoup4_view(): def pkg_beautifulsoup4_propagation_view(): from bs4 import BeautifulSoup - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_cachetools.py b/tests/appsec/iast_packages/packages/pkg_cachetools.py index 53805009867..5c3ef483f10 100644 --- a/tests/appsec/iast_packages/packages/pkg_cachetools.py +++ b/tests/appsec/iast_packages/packages/pkg_cachetools.py @@ -50,7 +50,7 @@ def expensive_function(key): def pkg_cachetools_propagation_view(): import cachetools - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) diff --git a/tests/appsec/iast_packages/packages/pkg_chartset_normalizer.py b/tests/appsec/iast_packages/packages/pkg_chartset_normalizer.py index e98d3547ad3..073ec6dd1f9 100644 --- a/tests/appsec/iast_packages/packages/pkg_chartset_normalizer.py +++ b/tests/appsec/iast_packages/packages/pkg_chartset_normalizer.py @@ -25,7 +25,7 @@ def pkg_charset_normalizer_view(): def pkg_charset_normalizer_propagation_view(): from charset_normalizer import from_bytes - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_cryptography.py b/tests/appsec/iast_packages/packages/pkg_cryptography.py index cf34079fce7..79e020e5863 100644 --- a/tests/appsec/iast_packages/packages/pkg_cryptography.py +++ b/tests/appsec/iast_packages/packages/pkg_cryptography.py @@ -41,7 +41,7 @@ def pkg_cryptography_view(): def pkg_cryptography_propagation_view(): from cryptography.fernet import Fernet - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_docutils.py b/tests/appsec/iast_packages/packages/pkg_docutils.py index 971fbacdd50..ff218f8fc17 100644 --- a/tests/appsec/iast_packages/packages/pkg_docutils.py +++ b/tests/appsec/iast_packages/packages/pkg_docutils.py @@ -43,7 +43,7 @@ def pkg_docutils_view(): def pkg_docutils_propagation_view(): import docutils.core - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) diff --git a/tests/appsec/iast_packages/packages/pkg_exceptiongroup.py b/tests/appsec/iast_packages/packages/pkg_exceptiongroup.py index 4f1786237e2..2aa092304c3 100644 --- a/tests/appsec/iast_packages/packages/pkg_exceptiongroup.py +++ b/tests/appsec/iast_packages/packages/pkg_exceptiongroup.py @@ -46,7 +46,7 @@ def raise_exceptions(param): def pkg_exceptiongroup_propagation_view(): from exceptiongroup import ExceptionGroup - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) try: diff --git a/tests/appsec/iast_packages/packages/pkg_idna.py b/tests/appsec/iast_packages/packages/pkg_idna.py index 1421d5c2dcf..19ace566034 100644 --- a/tests/appsec/iast_packages/packages/pkg_idna.py +++ b/tests/appsec/iast_packages/packages/pkg_idna.py @@ -27,7 +27,7 @@ def pkg_idna_view(): def pkg_idna_propagation_view(): import idna - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_iniconfig.py b/tests/appsec/iast_packages/packages/pkg_iniconfig.py index 4f204d7ee54..8ecf2c52b98 100644 --- a/tests/appsec/iast_packages/packages/pkg_iniconfig.py +++ b/tests/appsec/iast_packages/packages/pkg_iniconfig.py @@ -50,7 +50,7 @@ def pkg_iniconfig_view(): def pkg_iniconfig_propagation_view(): import iniconfig - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) try: diff --git a/tests/appsec/iast_packages/packages/pkg_jinja2.py b/tests/appsec/iast_packages/packages/pkg_jinja2.py index acedfff0d1e..4699d2f85a2 100644 --- a/tests/appsec/iast_packages/packages/pkg_jinja2.py +++ b/tests/appsec/iast_packages/packages/pkg_jinja2.py @@ -36,7 +36,7 @@ def pkg_jinja2_view(): def pkg_jinja2_propagation_view(): from jinja2 import Template - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_lxml.py b/tests/appsec/iast_packages/packages/pkg_lxml.py index 3309731f197..81a59e5758b 100644 --- a/tests/appsec/iast_packages/packages/pkg_lxml.py +++ b/tests/appsec/iast_packages/packages/pkg_lxml.py @@ -39,7 +39,7 @@ def pkg_lxml_view(): def pkg_lxml_propagation_view(): from lxml import etree - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_multidict.py b/tests/appsec/iast_packages/packages/pkg_multidict.py index f0cbe10f028..b07a5c69731 100644 --- a/tests/appsec/iast_packages/packages/pkg_multidict.py +++ b/tests/appsec/iast_packages/packages/pkg_multidict.py @@ -36,7 +36,7 @@ def pkg_multidict_view(): def pkg_multidict_propagation_view(): from multidict import MultiDict - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_platformdirs.py b/tests/appsec/iast_packages/packages/pkg_platformdirs.py index 838c0a20e01..325a0177cbf 100644 --- a/tests/appsec/iast_packages/packages/pkg_platformdirs.py +++ b/tests/appsec/iast_packages/packages/pkg_platformdirs.py @@ -47,7 +47,7 @@ def pkg_platformdirs_view(): def pkg_platformdirs_propagation_view(): from platformdirs import user_data_dir - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_pyasn1.py b/tests/appsec/iast_packages/packages/pkg_pyasn1.py index 8e64024ad3c..3f870889f07 100644 --- a/tests/appsec/iast_packages/packages/pkg_pyasn1.py +++ b/tests/appsec/iast_packages/packages/pkg_pyasn1.py @@ -52,7 +52,7 @@ def pkg_pyasn1_propagation_view(): from pyasn1.type import namedtype from pyasn1.type import univ - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_pygments.py b/tests/appsec/iast_packages/packages/pkg_pygments.py index 6cde162a6bd..8466cb40333 100644 --- a/tests/appsec/iast_packages/packages/pkg_pygments.py +++ b/tests/appsec/iast_packages/packages/pkg_pygments.py @@ -45,7 +45,7 @@ def pkg_pygments_propagation_view(): from pygments.formatters import HtmlFormatter from pygments.lexers import PythonLexer - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) diff --git a/tests/appsec/iast_packages/packages/pkg_pynacl.py b/tests/appsec/iast_packages/packages/pkg_pynacl.py index 78c8baadb3a..1e8435d050f 100644 --- a/tests/appsec/iast_packages/packages/pkg_pynacl.py +++ b/tests/appsec/iast_packages/packages/pkg_pynacl.py @@ -53,7 +53,7 @@ def pkg_pynacl_propagation_view(): from nacl import secret from nacl import utils - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) diff --git a/tests/appsec/iast_packages/packages/pkg_pyparsing.py b/tests/appsec/iast_packages/packages/pkg_pyparsing.py index bcc1647adb7..b5defd3c2ec 100644 --- a/tests/appsec/iast_packages/packages/pkg_pyparsing.py +++ b/tests/appsec/iast_packages/packages/pkg_pyparsing.py @@ -47,7 +47,7 @@ def pkg_pyparsing_view(): def pkg_pyparsing_propagation_view(): import pyparsing as pp - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) diff --git a/tests/appsec/iast_packages/packages/pkg_python_multipart.py b/tests/appsec/iast_packages/packages/pkg_python_multipart.py index b8fab3ec159..0f0b79e5b85 100644 --- a/tests/appsec/iast_packages/packages/pkg_python_multipart.py +++ b/tests/appsec/iast_packages/packages/pkg_python_multipart.py @@ -33,7 +33,7 @@ def pkg_multipart_view(): def pkg_multipart_propagation_view(): from multipart.multipart import parse_options_header - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_pyyaml.py b/tests/appsec/iast_packages/packages/pkg_pyyaml.py index 7d394c998f3..93a37976ed1 100644 --- a/tests/appsec/iast_packages/packages/pkg_pyyaml.py +++ b/tests/appsec/iast_packages/packages/pkg_pyyaml.py @@ -30,7 +30,7 @@ def pkg_pyyaml_view(): def pkg_pyyaml_propagation_view(): import yaml - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_rsa.py b/tests/appsec/iast_packages/packages/pkg_rsa.py index 209b2aef783..b921f18c33a 100644 --- a/tests/appsec/iast_packages/packages/pkg_rsa.py +++ b/tests/appsec/iast_packages/packages/pkg_rsa.py @@ -36,7 +36,7 @@ def pkg_rsa_view(): def pkg_rsa_propagation_view(): import rsa - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_soupsieve.py b/tests/appsec/iast_packages/packages/pkg_soupsieve.py index a4017bdcde2..eba3539e318 100644 --- a/tests/appsec/iast_packages/packages/pkg_soupsieve.py +++ b/tests/appsec/iast_packages/packages/pkg_soupsieve.py @@ -43,7 +43,7 @@ def pkg_soupsieve_propagation_view(): from bs4 import BeautifulSoup import soupsieve as sv - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_sqlalchemy.py b/tests/appsec/iast_packages/packages/pkg_sqlalchemy.py index 09fe47336fb..17daa4b5405 100644 --- a/tests/appsec/iast_packages/packages/pkg_sqlalchemy.py +++ b/tests/appsec/iast_packages/packages/pkg_sqlalchemy.py @@ -58,7 +58,7 @@ def pkg_sqlalchemy_propagation_view(): from sqlalchemy import create_engine from sqlalchemy.orm import declarative_base - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_tomli.py b/tests/appsec/iast_packages/packages/pkg_tomli.py index 7741aeea8bc..f21aa2e8a54 100644 --- a/tests/appsec/iast_packages/packages/pkg_tomli.py +++ b/tests/appsec/iast_packages/packages/pkg_tomli.py @@ -38,7 +38,7 @@ def pkg_tomli_view(): def pkg_tomli_propagation_view(): import tomli - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/pkg_wrapt.py b/tests/appsec/iast_packages/packages/pkg_wrapt.py index be624bc399b..a58504f294c 100644 --- a/tests/appsec/iast_packages/packages/pkg_wrapt.py +++ b/tests/appsec/iast_packages/packages/pkg_wrapt.py @@ -46,7 +46,7 @@ def sample_function(param): @pkg_wrapt.route("/wrapt_propagation") def pkg_wrapt_propagation_view(): - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) diff --git a/tests/appsec/iast_packages/packages/pkg_yarl.py b/tests/appsec/iast_packages/packages/pkg_yarl.py index 02940a240ea..9e254aab2c7 100644 --- a/tests/appsec/iast_packages/packages/pkg_yarl.py +++ b/tests/appsec/iast_packages/packages/pkg_yarl.py @@ -44,7 +44,7 @@ def pkg_yarl_view(): def pkg_yarl_propagation_view(): from yarl import URL - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted response = ResultResponse(request.args.get("package_param")) if not is_pyobject_tainted(response.package_param): diff --git a/tests/appsec/iast_packages/packages/utils.py b/tests/appsec/iast_packages/packages/utils.py index c36c6966f9e..7c6d4c95f5b 100644 --- a/tests/appsec/iast_packages/packages/utils.py +++ b/tests/appsec/iast_packages/packages/utils.py @@ -2,7 +2,7 @@ with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted class ResultResponse: diff --git a/tests/appsec/iast_tdd_propagation/flask_orm_app.py b/tests/appsec/iast_tdd_propagation/flask_orm_app.py index b7fcf9f59c2..b4e7e0d2095 100644 --- a/tests/appsec/iast_tdd_propagation/flask_orm_app.py +++ b/tests/appsec/iast_tdd_propagation/flask_orm_app.py @@ -18,7 +18,7 @@ with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted import ddtrace.auto # noqa: F401 # isort: skip diff --git a/tests/appsec/iast_tdd_propagation/flask_propagation_views.py b/tests/appsec/iast_tdd_propagation/flask_propagation_views.py index 0cf9f201d7f..ae1ce2af489 100644 --- a/tests/appsec/iast_tdd_propagation/flask_propagation_views.py +++ b/tests/appsec/iast_tdd_propagation/flask_propagation_views.py @@ -4,7 +4,7 @@ from flask import request from ddtrace import tracer -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted class ResultResponse: diff --git a/tests/appsec/iast_tdd_propagation/flask_taint_sinks_views.py b/tests/appsec/iast_tdd_propagation/flask_taint_sinks_views.py index 56074989bc5..396aa0db63c 100644 --- a/tests/appsec/iast_tdd_propagation/flask_taint_sinks_views.py +++ b/tests/appsec/iast_tdd_propagation/flask_taint_sinks_views.py @@ -6,7 +6,7 @@ from flask import request from ddtrace import tracer -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from tests.appsec.iast.taint_sinks.conftest import _get_span_report diff --git a/tests/appsec/integrations/pygoat_tests/test_pygoat.py b/tests/appsec/integrations/pygoat_tests/test_pygoat.py index f3dd0f173ee..8bb8baae1bd 100644 --- a/tests/appsec/integrations/pygoat_tests/test_pygoat.py +++ b/tests/appsec/integrations/pygoat_tests/test_pygoat.py @@ -143,7 +143,7 @@ def test_sqli(client): @pytest.mark.skip("TODO: SSRF is not implemented for open()") def test_ssrf1(client, iast_context_defaults): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject s = "templates/Lab/ssrf/blogs/blog2.txt" tainted_path = taint_pyobject( @@ -160,7 +160,7 @@ def test_ssrf1(client, iast_context_defaults): def test_ssrf2(client, iast_context_defaults): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject s = "http://example.com" tainted_path = taint_pyobject( diff --git a/tests/appsec/integrations/test_flask_iast_patching.py b/tests/appsec/integrations/test_flask_iast_patching.py index 5dd1baab67c..3291297ea92 100644 --- a/tests/appsec/integrations/test_flask_iast_patching.py +++ b/tests/appsec/integrations/test_flask_iast_patching.py @@ -18,7 +18,7 @@ def test_flask_iast_ast_patching_import_error(): pass """ with flask_server( - appsec_enabled="false", iast_enabled="true", token=None, port=_PORT, assert_debug=True + appsec_enabled="false", iast_enabled="true", token=None, port=_PORT, assert_debug=False ) as context: _, flask_client, pid = context diff --git a/tests/appsec/integrations/test_langchain.py b/tests/appsec/integrations/test_langchain.py index 795d48db8b9..cf0dca1e49e 100644 --- a/tests/appsec/integrations/test_langchain.py +++ b/tests/appsec/integrations/test_langchain.py @@ -14,7 +14,7 @@ with override_env({"DD_IAST_ENABLED": "True"}): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject @pytest.mark.skipif(not is_module_installed("langchain"), reason="Langchain tests work on 3.9 or higher") diff --git a/tests/appsec/integrations/test_psycopg2.py b/tests/appsec/integrations/test_psycopg2.py index 3e08670f2d1..d6d25f7ffc2 100644 --- a/tests/appsec/integrations/test_psycopg2.py +++ b/tests/appsec/integrations/test_psycopg2.py @@ -2,7 +2,7 @@ import pytest from ddtrace.appsec._iast._taint_tracking import OriginType -from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_utils import LazyTaintList from tests.appsec.iast.conftest import _end_iast_context_and_oce from tests.appsec.iast.conftest import _start_iast_context_and_oce diff --git a/tests/ci_visibility/api_client/_util.py b/tests/ci_visibility/api_client/_util.py index 8a260fbf3e6..403482688a9 100644 --- a/tests/ci_visibility/api_client/_util.py +++ b/tests/ci_visibility/api_client/_util.py @@ -105,6 +105,23 @@ def _get_tests_api_response(tests_body: t.Optional[t.Dict] = None): return Response(200, json.dumps(response)) +def _get_detailed_tests_api_response(modules: t.Dict): + response = {"data": {"id": "J0ucvcSApX8", "type": "ci_app_libraries_tests", "attributes": {"modules": []}}} + + for module_id, suites in modules.items(): + module = {"id": module_id, "suites": []} + response["data"]["attributes"]["modules"].append(module) + + for suite_id, tests in suites.items(): + suite = {"id": suite_id, "tests": []} + module["suites"].append(suite) + + for test_id in tests: + suite["tests"].append({"id": test_id}) + + return Response(200, json.dumps(response)) + + def _make_fqdn_internal_test_id(module_name: str, suite_name: str, test_name: str, parameters: t.Optional[str] = None): """An easy way to create a test id "from the bottom up" diff --git a/tests/ci_visibility/test_encoder.py b/tests/ci_visibility/test_encoder.py index 402668a624a..4a1d0fe2331 100644 --- a/tests/ci_visibility/test_encoder.py +++ b/tests/ci_visibility/test_encoder.py @@ -1,15 +1,9 @@ import json import os -from unittest import mock import msgpack -import pytest -import ddtrace from ddtrace._trace.span import Span -from ddtrace.contrib.pytest.plugin import is_enabled -from ddtrace.internal.ci_visibility import CIVisibility -from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import ITR_CORRELATION_ID_TAG_NAME from ddtrace.internal.ci_visibility.constants import SESSION_ID @@ -17,10 +11,7 @@ from ddtrace.internal.ci_visibility.encoder import CIVisibilityCoverageEncoderV02 from ddtrace.internal.ci_visibility.encoder import CIVisibilityEncoderV01 from ddtrace.internal.encoding import JSONEncoder -from tests.ci_visibility.test_ci_visibility import _dummy_noop_git_client -from tests.ci_visibility.util import _patch_dummy_writer -from tests.utils import TracerTestCase -from tests.utils import override_env +from tests.contrib.pytest.test_pytest import PytestTestCaseBase def test_encode_traces_civisibility_v0(): @@ -241,38 +232,7 @@ def test_encode_traces_civisibility_v2_coverage_empty_traces(): assert complete_payload is None -class PytestEncodingTestCase(TracerTestCase): - @pytest.fixture(autouse=True) - def fixtures(self, testdir, monkeypatch): - self.testdir = testdir - self.monkeypatch = monkeypatch - - def inline_run(self, *args): - """Execute test script with test tracer.""" - - class CIVisibilityPlugin: - @staticmethod - def pytest_configure(config): - if is_enabled(config): - with _patch_dummy_writer(): - assert CIVisibility.enabled - CIVisibility.disable() - CIVisibility.enable(tracer=self.tracer, config=ddtrace.config.pytest) - - with override_env(dict(DD_API_KEY="foobar.baz")), _dummy_noop_git_client(), mock.patch( - "ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings", - return_value=TestVisibilityAPISettings(False, False, False, False), - ): - return self.testdir.inline_run(*args, plugins=[CIVisibilityPlugin()]) - - def subprocess_run(self, *args): - """Execute test script with test tracer.""" - return self.testdir.runpytest_subprocess(*args) - - def teardown(self): - if CIVisibility.enabled: - CIVisibility.disable() - +class PytestEncodingTestCase(PytestTestCaseBase): def test_event_payload(self): """Test that a pytest test case will generate a test event, but with: - test_session_id, test_module_id, test_suite_id moved from meta to event content dictionary diff --git a/tests/ci_visibility/test_quarantine.py b/tests/ci_visibility/test_quarantine.py new file mode 100644 index 00000000000..b467431d074 --- /dev/null +++ b/tests/ci_visibility/test_quarantine.py @@ -0,0 +1,56 @@ +from pathlib import Path +from unittest import mock + +from ddtrace.ext.test_visibility.api import TestStatus +from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings +from ddtrace.internal.ci_visibility.api._base import TestVisibilitySessionSettings +from ddtrace.internal.ci_visibility.api._session import TestVisibilitySession +from ddtrace.internal.ci_visibility.api._test import TestVisibilityTest +from ddtrace.internal.ci_visibility.telemetry.constants import TEST_FRAMEWORKS +from ddtrace.internal.test_visibility._atr_mixins import AutoTestRetriesSettings +from tests.utils import DummyTracer + + +class TestCIVisibilityTestQuarantine: + """Tests that the classes in the CIVisibility API correctly handle quarantine.""" + + def _get_session_settings( + self, + ) -> TestVisibilitySessionSettings: + return TestVisibilitySessionSettings( + tracer=DummyTracer(), + test_service="qurantine_test_service", + test_command="qurantine_test_command", + test_framework="qurantine_test_framework", + test_framework_metric_name=TEST_FRAMEWORKS.MANUAL, + test_framework_version="0.0", + session_operation_name="qurantine_session", + module_operation_name="qurantine_module", + suite_operation_name="qurantine_suite", + test_operation_name="qurantine_test", + workspace_path=Path().absolute(), + efd_settings=EarlyFlakeDetectionSettings(enabled=False), + atr_settings=AutoTestRetriesSettings(enabled=False), + quarantine_settings=QuarantineSettings(enabled=True), + ) + + def test_quarantine_tags_set(self): + session = TestVisibilitySession( + session_settings=self._get_session_settings(), + ) + + test = TestVisibilityTest( + name="quarantine_test_1", + session_settings=session._session_settings, + is_quarantined=True, + ) + + with mock.patch.object(test, "get_session", return_value=session): + session.start() + test.start() + test.finish_test(TestStatus.FAIL) + session.finish() + + assert test._span.get_tag("test.quarantine.is_quarantined") == "true" + assert session._span.get_tag("test_session.quarantine.enabled") == "true" diff --git a/tests/contrib/asgi/test_asgi.py b/tests/contrib/asgi/test_asgi.py index 40935990fc2..c1e1f1c7328 100644 --- a/tests/contrib/asgi/test_asgi.py +++ b/tests/contrib/asgi/test_asgi.py @@ -1,5 +1,6 @@ import asyncio from functools import partial +import logging import os import random @@ -10,6 +11,7 @@ from ddtrace.constants import ERROR_MSG from ddtrace.contrib.asgi import TraceMiddleware from ddtrace.contrib.asgi import span_from_scope +from ddtrace.contrib.internal.asgi.middleware import _parse_response_cookies from ddtrace.propagation import http as http_propagation from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME from tests.utils import DummyTracer @@ -634,6 +636,52 @@ async def test_tasks_asgi_without_more_body(scope, tracer, test_spans): assert request_span.duration < 1 +@pytest.mark.asyncio +async def test_request_parse_response_cookies(tracer, test_spans, caplog): + """ + Regression test https://github.com/DataDog/dd-trace-py/issues/11818 + """ + + async def tasks_cookies(scope, receive, send): + message = await receive() + if message.get("type") == "http.request": + await send({"type": "http.response.start", "status": 200, "headers": [[b"set-cookie", b"test_cookie"]]}) + await send({"type": "http.response.body", "body": b"*"}) + await asyncio.sleep(1) + + with caplog.at_level(logging.DEBUG): + app = TraceMiddleware(tasks_cookies, tracer=tracer) + async with httpx.AsyncClient(app=app) as client: + response = await client.get("http://testserver/") + assert response.status_code == 200 + + assert "failed to extract response cookies" not in caplog.text + + +@pytest.mark.parametrize( + "headers,expected_result", + [ + ({}, {}), + ({"cookie": "cookie1=value1"}, {}), + ({"header-1": ""}, {}), + ({"Set-cookie": "cookie1=value1"}, {}), + ({"set-Cookie": "cookie1=value1"}, {}), + ({"SET-cookie": "cookie1=value1"}, {}), + ({"set-cookie": "a"}, {}), + ({"set-cookie": "1234"}, {}), + ({"set-cookie": "cookie1=value1"}, {"cookie1": "value1"}), + ({"set-cookie": "cookie2=value1=value2"}, {"cookie2": "value1=value2"}), + ({"set-cookie": "cookie3=="}, {"cookie3": "="}), + ], +) +def test__parse_response_cookies(headers, expected_result, caplog): + with caplog.at_level(logging.DEBUG): + result = _parse_response_cookies(headers) + + assert "failed to extract response cookies" not in caplog.text + assert result == expected_result + + @pytest.mark.asyncio async def test_tasks_asgi_with_more_body(scope, tracer, test_spans): """ diff --git a/tests/contrib/dbapi/test_dbapi_appsec.py b/tests/contrib/dbapi/test_dbapi_appsec.py index 0ee86f99685..b60b3ac05c0 100644 --- a/tests/contrib/dbapi/test_dbapi_appsec.py +++ b/tests/contrib/dbapi/test_dbapi_appsec.py @@ -36,7 +36,7 @@ def tearDown(self): @pytest.mark.skipif(not _is_python_version_supported(), reason="IAST compatible versions") def test_tainted_query(self): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject with override_global_config( dict( @@ -59,7 +59,7 @@ def test_tainted_query(self): @pytest.mark.skipif(not _is_python_version_supported(), reason="IAST compatible versions") def test_tainted_query_args(self): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject with mock.patch( "ddtrace.appsec._iast.taint_sinks.sql_injection.SqlInjection.report" @@ -113,7 +113,7 @@ def test_untainted_query_and_args(self): @pytest.mark.skipif(not _is_python_version_supported(), reason="IAST compatible versions") def test_tainted_query_iast_disabled(self): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import taint_pyobject + from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject with mock.patch( "ddtrace.appsec._iast.taint_sinks.sql_injection.SqlInjection.report" diff --git a/tests/contrib/django/django_app/appsec_urls.py b/tests/contrib/django/django_app/appsec_urls.py index 7ded023460f..f5b3f359445 100644 --- a/tests/contrib/django/django_app/appsec_urls.py +++ b/tests/contrib/django/django_app/appsec_urls.py @@ -9,19 +9,13 @@ from ddtrace import tracer from ddtrace.appsec import _asm_request_context +from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect +from ddtrace.appsec._iast._taint_tracking.aspects import decode_aspect from ddtrace.appsec._iast._utils import _is_python_version_supported as python_supported_by_iast from ddtrace.appsec._trace_utils import block_request_if_user_blocked from tests.utils import override_env -try: - with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect - from ddtrace.appsec._iast._taint_tracking.aspects import decode_aspect -except ImportError: - # Python 2 compatibility - from operator import add as add_aspect - # django.conf.urls.url was deprecated in django 3 and removed in django 4 if django.VERSION < (4, 0, 0): from django.conf.urls import url as handler @@ -80,9 +74,14 @@ def checkuser_view(request, user_id): def sqli_http_request_parameter(request): + import bcrypt + from django.contrib.auth.hashers import BCryptSHA256PasswordHasher + + password_django = BCryptSHA256PasswordHasher() + obj = password_django.encode("i'm a password", bcrypt.gensalt()) with connection.cursor() as cursor: # label iast_enabled_sqli_http_request_parameter - cursor.execute(request.GET["q"]) + cursor.execute(add_aspect(add_aspect(request.GET["q"], obj), "'")) return HttpResponse(request.META["HTTP_USER_AGENT"], status=200) @@ -123,7 +122,7 @@ def taint_checking_enabled_view(request): if python_supported_by_iast(): with override_env({"DD_IAST_ENABLED": "True"}): from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast.reporter import IastSpanReporter def assert_origin_path(path): # type: (Any) -> None @@ -155,7 +154,7 @@ def is_pyobject_tainted(pyobject): # type: (Any) -> bool def taint_checking_disabled_view(request): if python_supported_by_iast(): with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted else: def is_pyobject_tainted(pyobject): # type: (Any) -> bool diff --git a/tests/contrib/django/test_django_appsec_iast.py b/tests/contrib/django/test_django_appsec_iast.py index efe0fa9acd0..7e42e8aa903 100644 --- a/tests/contrib/django/test_django_appsec_iast.py +++ b/tests/contrib/django/test_django_appsec_iast.py @@ -204,14 +204,14 @@ def test_django_tainted_user_agent_iast_disabled(client, test_spans, tracer): @pytest.mark.django_db() @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter(client, test_spans, tracer): - with override_global_config(dict(_iast_enabled=True)): + with override_global_config(dict(_iast_enabled=True, _deduplication_enabled=False, _iast_request_sampling=100.0)): root_span, response = _aux_appsec_get_root_span( client, test_spans, tracer, payload=urlencode({"mytestingbody_key": "mytestingbody_value"}), content_type="application/x-www-form-urlencoded", - url="/appsec/sqli_http_request_parameter/?q=SELECT 1 FROM sqlite_master", + url="/appsec/sqli_http_request_parameter/?q=SELECT 1 FROM sqlite_master WHERE name='", headers={"HTTP_USER_AGENT": "test/1.2.3"}, ) @@ -228,7 +228,7 @@ def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter(clie { "name": "q", "origin": "http.request.parameter", - "pattern": "abcdefghijklmnopqrstuvwxyzA", + "pattern": "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN", "redacted": True, } ] @@ -238,7 +238,9 @@ def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter(clie "valueParts": [ {"source": 0, "value": "SELECT "}, {"pattern": "h", "redacted": True, "source": 0}, - {"source": 0, "value": " FROM sqlite_master"}, + {"source": 0, "value": " FROM sqlite_master WHERE name='"}, + {"redacted": True}, + {"value": "'"}, ] } assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE diff --git a/tests/contrib/fastapi/test_fastapi_appsec_iast.py b/tests/contrib/fastapi/test_fastapi_appsec_iast.py index 7f1a140ffc2..1a5db995af4 100644 --- a/tests/contrib/fastapi/test_fastapi_appsec_iast.py +++ b/tests/contrib/fastapi/test_fastapi_appsec_iast.py @@ -74,8 +74,8 @@ def check_native_code_exception_in_each_fastapi_test(request, caplog, telemetry_ def test_query_param_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) @@ -109,8 +109,8 @@ async def test_route(request: Request): def test_header_value_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.headers.get("iast_header") ranges_result = get_tainted_ranges(query_params) @@ -146,8 +146,8 @@ async def test_route(request: Request): def test_header_value_source_typing_param(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(iast_header: typing.Annotated[str, Header()] = None): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(iast_header) @@ -180,8 +180,8 @@ async def test_route(iast_header: typing.Annotated[str, Header()] = None): def test_cookies_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.cookies.get("iast_cookie") ranges_result = get_tainted_ranges(query_params) @@ -216,8 +216,8 @@ async def test_route(request: Request): def test_cookies_source_typing_param(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html") async def test_route(iast_cookie: typing.Annotated[str, Cookie()] = "ddd"): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(iast_cookie) @@ -250,8 +250,8 @@ async def test_route(iast_cookie: typing.Annotated[str, Cookie()] = "ddd"): def test_path_param_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/index.html/{item_id}") async def test_route(item_id): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(item_id) @@ -283,8 +283,8 @@ async def test_route(item_id): def test_path_source(fastapi_application, client, tracer, test_spans): @fastapi_application.get("/path_source/") async def test_route(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges path = request.url.path ranges_result = get_tainted_ranges(path) @@ -317,8 +317,8 @@ async def test_route(request: Request): def test_path_body_receive_source(fastapi_application, client, tracer, test_spans): @fastapi_application.post("/index.html") async def test_route(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges body = await request.receive() result = body["body"] @@ -354,8 +354,8 @@ async def test_route(request: Request): def test_path_body_body_source(fastapi_application, client, tracer, test_spans): @fastapi_application.post("/index.html") async def test_route(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges body = await request.body() ranges_result = get_tainted_ranges(body) @@ -392,8 +392,8 @@ async def test_route(request: Request): def test_path_body_body_source_formdata_latest(fastapi_application, client, tracer, test_spans): @fastapi_application.post("/index.html") async def test_route(path: typing.Annotated[str, Form()]): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(path) @@ -423,8 +423,8 @@ async def test_route(path: typing.Annotated[str, Form()]): def test_path_body_body_source_formdata_90(fastapi_application, client, tracer, test_spans): @fastapi_application.post("/index.html") async def test_route(path: str = Form(...)): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(path) @@ -463,8 +463,8 @@ class Item(BaseModel): @fastapi_application.post("/index") async def test_route(item: Item): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(item.name) @@ -497,7 +497,7 @@ async def test_route(item: Item): def test_path_body_body_upload(fastapi_application, client, tracer, test_spans): @fastapi_application.post("/uploadfile/") async def create_upload_file(files: typing.List[UploadFile]): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges ranges_result = get_tainted_ranges(files[0]) return JSONResponse( @@ -529,7 +529,7 @@ def test_fastapi_sqli_path_param(fastapi_application, client, tracer, test_spans async def test_route(param_str): import sqlite3 - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect assert is_pyobject_tainted(param_str) @@ -577,8 +577,8 @@ async def test_route(param_str): def test_fasapi_insecure_cookie(fastapi_application, client, tracer, test_spans): @fastapi_application.route("/insecure_cookie/", methods=["GET"]) def insecure_cookie(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) @@ -618,8 +618,8 @@ def insecure_cookie(request: Request): def test_fasapi_insecure_cookie_empty(fastapi_application, client, tracer, test_spans): @fastapi_application.route("/insecure_cookie/", methods=["GET"]) def insecure_cookie(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) @@ -653,8 +653,8 @@ def insecure_cookie(request: Request): def test_fasapi_no_http_only_cookie(fastapi_application, client, tracer, test_spans): @fastapi_application.route("/insecure_cookie/", methods=["GET"]) def insecure_cookie(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) @@ -694,8 +694,8 @@ def insecure_cookie(request: Request): def test_fasapi_no_http_only_cookie_empty(fastapi_application, client, tracer, test_spans): @fastapi_application.route("/insecure_cookie/", methods=["GET"]) def insecure_cookie(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) @@ -729,8 +729,8 @@ def insecure_cookie(request: Request): def test_fasapi_no_samesite_cookie(fastapi_application, client, tracer, test_spans): @fastapi_application.route("/insecure_cookie/", methods=["GET"]) def insecure_cookie(request: Request): - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges from ddtrace.appsec._iast._taint_tracking import origin_to_str + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges query_params = request.query_params.get("iast_queryparam") ranges_result = get_tainted_ranges(query_params) diff --git a/tests/contrib/flask/test_flask_appsec_iast.py b/tests/contrib/flask/test_flask_appsec_iast.py index f1bed61cb9d..238d0630549 100644 --- a/tests/contrib/flask/test_flask_appsec_iast.py +++ b/tests/contrib/flask/test_flask_appsec_iast.py @@ -59,7 +59,7 @@ def test_flask_full_sqli_iast_http_request_path_parameter(self): def sqli_1(param_str): import sqlite3 - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect assert is_pyobject_tainted(param_str) @@ -273,8 +273,8 @@ def sqli_5(param_str, param_int): from flask import request from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking import get_tainted_ranges - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import get_tainted_ranges + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted header_ranges = get_tainted_ranges(request.headers["User-Agent"]) assert header_ranges @@ -324,7 +324,7 @@ def test_flask_simple_iast_path_header_and_querystring_tainted_request_sampling_ def sqli_6(param_str): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted # Note: these are not tainted because of request sampling at 0% assert not is_pyobject_tainted(request.headers["User-Agent"]) @@ -536,7 +536,7 @@ def sqli_10(): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect con = sqlite3.connect(":memory:") @@ -601,7 +601,7 @@ def sqli_11(): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect con = sqlite3.connect(":memory:") @@ -666,7 +666,7 @@ def sqli_11(): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect con = sqlite3.connect(":memory:") @@ -731,7 +731,7 @@ def sqli_11(): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect con = sqlite3.connect(":memory:") @@ -798,7 +798,7 @@ def sqli_11(): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect def iterate_json(data, parent_key=""): @@ -939,7 +939,7 @@ def sqli_10(): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect con = sqlite3.connect(":memory:") @@ -1042,7 +1042,7 @@ def header_injection(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1082,7 +1082,7 @@ def header_injection(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1111,7 +1111,7 @@ def header_injection(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1140,7 +1140,7 @@ def insecure_cookie(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1178,7 +1178,7 @@ def insecure_cookie_empty(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1208,7 +1208,7 @@ def no_http_only_cookie(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1246,7 +1246,7 @@ def no_http_only_cookie_empty(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1277,7 +1277,7 @@ def no_samesite_cookie(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1315,7 +1315,7 @@ def no_samesite_cookie_empty(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1343,7 +1343,7 @@ def cookie_secure(): from flask import Response from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted tainted_string = request.form.get("name") assert is_pyobject_tainted(tainted_string) @@ -1518,7 +1518,7 @@ def test_flask_simple_iast_path_header_and_querystring_not_tainted_if_iast_disab def test_sqli(param_str): from flask import request - from ddtrace.appsec._iast._taint_tracking import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted assert not is_pyobject_tainted(request.headers["User-Agent"]) assert not is_pyobject_tainted(request.query_string) diff --git a/tests/contrib/gevent/test_tracer.py b/tests/contrib/gevent/test_tracer.py index 9d622ac5d31..a40a0f3cdaf 100644 --- a/tests/contrib/gevent/test_tracer.py +++ b/tests/contrib/gevent/test_tracer.py @@ -411,6 +411,6 @@ def test_ddtracerun(self): p.wait() stdout, stderr = p.stdout.read(), p.stderr.read() - assert p.returncode == 0, stderr.decode() + assert p.returncode == 0, f"stdout: {stdout.decode()}\n\nstderr: {stderr.decode()}" assert b"Test success" in stdout, stdout.decode() assert b"RecursionError" not in stderr, stderr.decode() diff --git a/tests/contrib/pytest/test_pytest_atr.py b/tests/contrib/pytest/test_pytest_atr.py index 742a4f220d0..3e526e8cdf7 100644 --- a/tests/contrib/pytest/test_pytest_atr.py +++ b/tests/contrib/pytest/test_pytest_atr.py @@ -97,10 +97,6 @@ def set_up_atr(self): return_value=TestVisibilityAPISettings(flaky_test_retries_enabled=True), ): yield - from ddtrace.internal.ci_visibility.recorder import CIVisibility - - if CIVisibility.enabled: - CIVisibility.disable() def test_pytest_atr_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_pass=_TEST_PASS_CONTENT) diff --git a/tests/contrib/pytest/test_pytest_efd.py b/tests/contrib/pytest/test_pytest_efd.py index f08e170d089..2affcec3585 100644 --- a/tests/contrib/pytest/test_pytest_efd.py +++ b/tests/contrib/pytest/test_pytest_efd.py @@ -116,10 +116,6 @@ def set_up_efd(self): ), ): yield - from ddtrace.internal.ci_visibility.recorder import CIVisibility - - if CIVisibility.enabled: - CIVisibility.disable() def test_pytest_efd_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_known_pass=_TEST_KNOWN_PASS_CONTENT) diff --git a/tests/contrib/pytest/test_pytest_quarantine.py b/tests/contrib/pytest/test_pytest_quarantine.py new file mode 100644 index 00000000000..93b0b07eade --- /dev/null +++ b/tests/contrib/pytest/test_pytest_quarantine.py @@ -0,0 +1,444 @@ +"""Tests Early Flake Detection (EFD) functionality + +The tests in this module only validate the behavior of EFD, so only counts and statuses of tests, retries, and sessions +are checked. + +- The same known tests are used to override fetching of known tests. +- The session object is patched to never be a faulty session, by default. +""" +from unittest import mock + +import pytest + +from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.pytest._utils import _pytest_version_supports_efd +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings +from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings +from tests.contrib.pytest.test_pytest import PytestTestCaseBase +from tests.contrib.pytest.test_pytest import _get_spans_from_list + + +pytestmark = pytest.mark.skipif( + not (_USE_PLUGIN_V2 and _pytest_version_supports_efd()), + reason="Quarantine requires v2 of the plugin and pytest >=7.0", +) + +_TEST_PASS_QUARANTINED = """ +import pytest + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_pass_quarantined(): + assert True +""" + +_TEST_PASS_UNQUARANTINED = """ +import pytest + +def test_pass_normal(): + assert True +""" + +_TEST_FAIL_QUARANTINED = """ +import pytest + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_quarantined(): + assert False +""" + +_TEST_FAIL_UNQUARANTINED = """ +import pytest + +def test_fail_normal(): + assert False +""" + +_TEST_FAIL_SETUP_QUARANTINED = """ +import pytest + +@pytest.fixture() +def fail_setup(): + raise ValueError("fail setup") + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_setup(fail_setup): + assert True +""" + +_TEST_FAIL_TEARDOWN_QUARANTINED = """ +import pytest + +@pytest.fixture() +def fail_teardown(): + yield + raise ValueError("fail teardown") + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_teardown(fail_teardown): + assert True +""" + + +def assert_stats(rec, **outcomes): + """ + Assert that the correct number of test results of each type is present in a test run. + + This is similar to `rec.assertoutcome()`, but works with test statuses other than 'passed', 'failed' and 'skipped'. + """ + stats = {**rec.getcall("pytest_terminal_summary").terminalreporter.stats} + stats.pop("", None) + + for outcome, expected_count in outcomes.items(): + actual_count = len(stats.pop(outcome, [])) + assert actual_count == expected_count, f"Expected {expected_count} {outcome} tests, got {actual_count}" + + assert not stats, "Found unexpected stats in test results: {', '.join(stats.keys())}" + + +class PytestQuarantineTestCase(PytestTestCaseBase): + @pytest.fixture(autouse=True, scope="function") + def set_up_quarantine(self): + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=False, + ), + ): + yield + + def test_fail_quarantined_no_ddtrace_does_not_quarantine(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + rec = self.inline_run("-q") + assert rec.ret == 1 + assert_stats(rec, passed=2, failed=2) + assert len(self.pop_spans()) == 0 # ddtrace disabled, not collecting traces + + def test_fail_quarantined_with_ddtrace_does_not_fail_session(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + assert len(self.pop_spans()) > 0 + + def test_failing_and_passing_quarantined_and_unquarantined_tests(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 1 + assert_stats(rec, quarantined=2, passed=1, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_env_var_disables_quarantine(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q", extra_env={"DD_TEST_QUARANTINE_ENABLED": "0"}) + + assert rec.ret == 1 + assert_stats(rec, quarantined=0, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_env_var_does_not_override_api(self): + """Environment variable works as a kill-switch; if quarantine is disabled in the API, + the env var cannot make it enabled. + """ + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=False), + ), + ): + rec = self.inline_run("--ddtrace", "-q", extra_env={"DD_TEST_QUARANTINE_ENABLED": "1"}) + + assert rec.ret == 1 + assert_stats(rec, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_outcomes_without_atr(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "passed"), + ("call", "quarantined"), + ("teardown", "passed"), + ] + + assert len(rec.getcalls("pytest_pyfunc_call")) == 1 + + def test_quarantine_outcomes_with_atr(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=True, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "passed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_final_failed"), + ("teardown", "passed"), + ] + + assert len(rec.getcalls("pytest_pyfunc_call")) == 6 + + def test_quarantine_fail_setup(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_fail_teardown(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_TEARDOWN_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_spans_without_atr(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + [test_span_fail_quarantined] = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert test_span_fail_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_fail_quarantined.get_tag("test.status") == "fail" + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "pass" + + def test_quarantine_spans_with_atr(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=True, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + test_spans_fail_quarantined = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert len(test_spans_fail_quarantined) == 6 + assert all(span.get_tag("test.quarantine.is_quarantined") == "true" for span in test_spans_fail_quarantined) + assert all(span.get_tag("test.status") == "fail" for span in test_spans_fail_quarantined) + assert test_spans_fail_quarantined[0].get_tag("test.is_retry") is None + assert all(span.get_tag("test.is_retry") for span in test_spans_fail_quarantined[1:]) + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "pass" + + +class PytestQuarantineSkippingTestCase(PytestTestCaseBase): + @pytest.fixture(autouse=True, scope="function") + def set_up_quarantine(self): + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True, skip_quarantined_tests=True), + flaky_test_retries_enabled=False, + ), + ): + yield + + def test_fail_quarantined_no_ddtrace_does_not_quarantine(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + rec = self.inline_run("-q") + assert rec.ret == 1 + assert_stats(rec, passed=2, failed=2) + assert len(self.pop_spans()) == 0 # ddtrace disabled, not collecting traces + + def test_fail_quarantined_with_ddtrace_does_not_fail_session(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + assert len(self.pop_spans()) > 0 + + def test_failing_and_passing_quarantined_and_unquarantined_tests(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 1 + assert_stats(rec, quarantined=2, passed=1, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_outcomes_without_atr(self): + return self._test_quarantine_outcomes(atr_enabled=False) + + def test_quarantine_outcomes_with_atr(self): + return self._test_quarantine_outcomes(atr_enabled=True) + + def _test_quarantine_outcomes(self, atr_enabled): + # ATR should not retry tests skipped by quarantine. + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True, skip_quarantined_tests=True), + flaky_test_retries_enabled=atr_enabled, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "skipped"), + ("teardown", "passed"), + ] + + assert len(rec.getcalls("pytest_pyfunc_call")) == 0 # test function is not called + + def test_quarantine_fail_setup(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_fail_teardown(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_TEARDOWN_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_skipping_spans_atr_disabled(self): + return self._test_quarantine_skipping_spans(atr_enabled=False) + + def test_quarantine_skipping_spans_atr_enabled(self): + return self._test_quarantine_skipping_spans(atr_enabled=True) + + def _test_quarantine_skipping_spans(self, atr_enabled): + # ATR should not affect spans for skipped quarantined tests. + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_setup_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + self.testdir.makepyfile(test_fail_teardown_quarantined=_TEST_FAIL_TEARDOWN_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True, skip_quarantined_tests=True), + flaky_test_retries_enabled=atr_enabled, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=4) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + [test_span_fail_quarantined] = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert test_span_fail_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_fail_quarantined.get_tag("test.status") == "skip" + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "skip" + + [test_span_fail_setup] = _get_spans_from_list(spans, "test", "test_fail_setup") + assert test_span_fail_setup.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_fail_setup.get_tag("test.status") == "skip" + + [test_span_fail_teardown] = _get_spans_from_list(spans, "test", "test_fail_teardown") + assert test_span_fail_teardown.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_fail_teardown.get_tag("test.status") == "skip" diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index 83a48ea1f48..8c5fbc72da2 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -645,7 +645,7 @@ suites: - '@gevent' - tests/contrib/gevent/* runner: riot - snapshot: true + snapshot: false graphene: paths: - '@bootstrap' diff --git a/tests/debugging/mocking.py b/tests/debugging/mocking.py index dee76125ba3..4446bce559c 100644 --- a/tests/debugging/mocking.py +++ b/tests/debugging/mocking.py @@ -2,6 +2,7 @@ from collections import Counter from contextlib import contextmanager import json +from time import monotonic from time import sleep from typing import Any from typing import Generator @@ -16,7 +17,6 @@ from ddtrace.debugging._probe.remoteconfig import _filter_by_env_and_version from ddtrace.debugging._signal.collector import SignalCollector from ddtrace.debugging._uploader import LogsIntakeUploaderV1 -from ddtrace.internal.compat import monotonic from tests.debugging.probe.test_status import DummyProbeStatusLogger diff --git a/tests/integration/test_sampling.py b/tests/integration/test_sampling.py index bb0d421a2d1..902b430bbc8 100644 --- a/tests/integration/test_sampling.py +++ b/tests/integration/test_sampling.py @@ -331,7 +331,7 @@ def test_rate_limiter_on_long_running_spans(tracer): """ tracer.configure(sampler=DatadogSampler(rate_limit=5)) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=1617333414): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=1617333414): span_m30 = tracer.trace(name="march 30") span_m30.start = 1622347257 # Mar 30 2021 span_m30.finish(1617333414) # April 2 2021 diff --git a/tests/profiling/collector/test_stack.py b/tests/profiling/collector/test_stack.py index 690be4b183c..84f3ad60ea6 100644 --- a/tests/profiling/collector/test_stack.py +++ b/tests/profiling/collector/test_stack.py @@ -254,13 +254,12 @@ def test_ignore_profiler_single(): @pytest.mark.skipif(not TESTING_GEVENT, reason="Not testing gevent") -@pytest.mark.subprocess(ddtrace_run=True) +@pytest.mark.subprocess(ddtrace_run=True, env=dict(DD_PROFILING_IGNORE_PROFILER="1", DD_PROFILING_API_TIMEOUT="0.1")) def test_ignore_profiler_gevent_task(): import gevent.monkey gevent.monkey.patch_all() - import os import time from ddtrace.profiling import collector # noqa:F401 @@ -282,28 +281,22 @@ def collect(self): _fib(22) return [] - for ignore in (True, False): - os.environ["DD_PROFILING_API_TIMEOUT"] = "0.1" - os.environ["DD_PROFILING_IGNORE_PROFILER"] = str(ignore) - p = profiler.Profiler() - p.start() - # This test is particularly useful with gevent enabled: create a test collector that run often and for long - # we're sure to catch it with the StackProfiler and that it's not ignored. - c = CollectorTest(p._profiler._recorder, interval=0.00001) - c.start() + p = profiler.Profiler() + p.start() + # This test is particularly useful with gevent enabled: create a test collector that run often and for long + # we're sure to catch it with the StackProfiler and that it's not ignored. + c = CollectorTest(p._profiler._recorder, interval=0.00001) + c.start() - for _ in range(100): - events = p._profiler._recorder.reset() - ids = {e.task_id for e in events[stack_event.StackSampleEvent]} - if (c._worker.ident in ids) != str(ignore): - break - # Give some time for gevent to switch greenlets - time.sleep(0.1) - else: - raise AssertionError("ignore == " + ignore) + for _ in range(100): + events = p._profiler._recorder.reset() + ids = {e.task_id for e in events[stack_event.StackSampleEvent]} + if c._worker.ident in ids: + raise AssertionError("Collector thread found") + time.sleep(0.1) - c.stop() - p.stop(flush=False) + c.stop() + p.stop(flush=False) def test_collect(): diff --git a/tests/profiling/exporter/test_http.py b/tests/profiling/exporter/test_http.py index 42e3aeb6fe6..26b69ff495e 100644 --- a/tests/profiling/exporter/test_http.py +++ b/tests/profiling/exporter/test_http.py @@ -201,14 +201,15 @@ def test_wrong_api_key(endpoint_test_server): @pytest.mark.subprocess(env=dict(DD_TRACE_AGENT_URL=_ENDPOINT)) def test_export(endpoint_test_server): - from ddtrace.internal import compat + import time + from ddtrace.profiling.exporter import http from tests.profiling.exporter import test_pprof from tests.profiling.exporter.test_http import _API_KEY from tests.profiling.exporter.test_http import _get_span_processor exp = http.PprofHTTPExporter(api_key=_API_KEY, endpoint_call_counter_span_processor=_get_span_processor()) - exp.export(test_pprof.TEST_EVENTS, 0, compat.time_ns()) + exp.export(test_pprof.TEST_EVENTS, 0, time.time_ns()) @pytest.mark.subprocess(env=dict(DD_TRACE_AGENT_URL="http://localhost:2")) diff --git a/tests/profiling/test_accuracy.py b/tests/profiling/test_accuracy.py index 7d94e725b72..a332068e12b 100644 --- a/tests/profiling/test_accuracy.py +++ b/tests/profiling/test_accuracy.py @@ -3,8 +3,6 @@ import pytest -from ddtrace.internal import compat - def spend_1(): time.sleep(1) @@ -33,16 +31,16 @@ def spend_16(): def spend_cpu_2(): - now = compat.monotonic_ns() + now = time.process_time_ns() # Active wait for 2 seconds - while compat.monotonic_ns() - now < 2e9: + while time.process_time_ns() - now < 2e9: pass def spend_cpu_3(): # Active wait for 3 seconds - now = compat.monotonic_ns() - while compat.monotonic_ns() - now < 3e9: + now = time.process_time_ns() + while time.process_time_ns() - now < 3e9: pass @@ -53,8 +51,12 @@ def spend_cpu_3(): CPU_TOLERANCE = 0.05 -def almost_equal(value, target, tolerance=TOLERANCE): - return abs(value - target) / target <= tolerance +def assert_almost_equal(value, target, tolerance=TOLERANCE): + if abs(value - target) / target > tolerance: + raise AssertionError( + f"Assertion failed: {value} is not approximately equal to {target} " + f"within tolerance={tolerance}, actual error={abs(value - target) / target}" + ) def total_time(time_data, funcname): @@ -68,7 +70,7 @@ def test_accuracy(): from ddtrace.profiling import profiler from ddtrace.profiling.collector import stack_event from tests.profiling.test_accuracy import CPU_TOLERANCE - from tests.profiling.test_accuracy import almost_equal + from tests.profiling.test_accuracy import assert_almost_equal from tests.profiling.test_accuracy import spend_16 from tests.profiling.test_accuracy import total_time @@ -87,20 +89,13 @@ def test_accuracy(): time_spent_ns[idx][frame[2]] += event.wall_time_ns cpu_spent_ns[idx][frame[2]] += event.cpu_time_ns - assert almost_equal(total_time(time_spent_ns, "spend_3"), 9e9) - assert almost_equal(total_time(time_spent_ns, "spend_1"), 2e9) - assert almost_equal(total_time(time_spent_ns, "spend_4"), 4e9) - assert almost_equal(total_time(time_spent_ns, "spend_16"), 16e9) - assert almost_equal(total_time(time_spent_ns, "spend_7"), 7e9) - - try: - from time import monotonic_ns # noqa:F401 - except ImportError: - # If we don't have access to high resolution clocks, we can't really test accurately things as it's spread in - # various Python implementation of monotonic, etc. - pass - else: - assert almost_equal(total_time(time_spent_ns, "spend_cpu_2"), 2e9) - assert almost_equal(total_time(time_spent_ns, "spend_cpu_3"), 3e9) - assert almost_equal(total_time(time_spent_ns, "spend_cpu_2"), 2e9, CPU_TOLERANCE) - assert almost_equal(total_time(time_spent_ns, "spend_cpu_3"), 3e9, CPU_TOLERANCE) + assert_almost_equal(total_time(time_spent_ns, "spend_3"), 9e9) + assert_almost_equal(total_time(time_spent_ns, "spend_1"), 2e9) + assert_almost_equal(total_time(time_spent_ns, "spend_4"), 4e9) + assert_almost_equal(total_time(time_spent_ns, "spend_16"), 16e9) + assert_almost_equal(total_time(time_spent_ns, "spend_7"), 7e9) + + assert_almost_equal(total_time(time_spent_ns, "spend_cpu_2"), 2e9) + assert_almost_equal(total_time(time_spent_ns, "spend_cpu_3"), 3e9) + assert_almost_equal(total_time(cpu_spent_ns, "spend_cpu_2"), 2e9, CPU_TOLERANCE) + assert_almost_equal(total_time(cpu_spent_ns, "spend_cpu_3"), 3e9, CPU_TOLERANCE) diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 483ec643527..42d8ccc29fe 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -1,9 +1,9 @@ # -*- encoding: utf-8 -*- import logging +import time import mock -from ddtrace.internal import compat from ddtrace.profiling import event from ddtrace.profiling import exporter from ddtrace.profiling import recorder @@ -64,11 +64,11 @@ def test_serverless_periodic(mock_periodic): r = recorder.Recorder() s = scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) # Fake start() - s._last_export = compat.time_ns() + s._last_export = time.time_ns() s.periodic() assert s._profiled_intervals == 1 mock_periodic.assert_not_called() - s._last_export = compat.time_ns() - 65 + s._last_export = time.time_ns() - 65 s._profiled_intervals = 65 s.periodic() assert s._profiled_intervals == 0 diff --git a/tests/profiling_v2/collector/test_stack.py b/tests/profiling_v2/collector/test_stack.py index af13a1ea237..74def22ed50 100644 --- a/tests/profiling_v2/collector/test_stack.py +++ b/tests/profiling_v2/collector/test_stack.py @@ -13,6 +13,7 @@ from ddtrace.profiling.collector import stack from ddtrace.settings.profiling import config from tests.profiling.collector import pprof_utils +from tests.profiling.collector import test_collector # Python 3.11.9 is not compatible with gevent, https://github.com/gevent/gevent/issues/2040 @@ -24,6 +25,43 @@ ) +# Use subprocess as ddup config persists across tests. +@pytest.mark.subprocess( + env=dict( + DD_PROFILING_MAX_FRAMES="5", + DD_PROFILING_OUTPUT_PPROF="/tmp/test_collect_truncate", + DD_PROFILING_STACK_V2_ENABLED="1", + ) +) +@pytest.mark.skipif(sys.version_info[:2] == (3, 7), reason="stack_v2 is not supported on Python 3.7") +def test_collect_truncate(): + import os + + from ddtrace.profiling import profiler + from tests.profiling.collector import pprof_utils + from tests.profiling.collector.test_stack import func1 + + pprof_prefix = os.environ["DD_PROFILING_OUTPUT_PPROF"] + output_filename = pprof_prefix + "." + str(os.getpid()) + + max_nframes = int(os.environ["DD_PROFILING_MAX_FRAMES"]) + + p = profiler.Profiler() + p.start() + + func1() + + p.stop() + + profile = pprof_utils.parse_profile(output_filename) + samples = pprof_utils.get_samples_with_value_type(profile, "wall-time") + assert len(samples) > 0 + for sample in samples: + # stack v2 adds one extra frame for "%d frames omitted" message + # Also, it allows max_nframes + 1 frames, so we add 2 here. + assert len(sample.location_id) <= max_nframes + 2, len(sample.location_id) + + @pytest.mark.parametrize("stack_v2_enabled", [True, False]) def test_stack_locations(stack_v2_enabled, tmp_path): if sys.version_info[:2] == (3, 7) and stack_v2_enabled: @@ -651,8 +689,23 @@ def _dofib(): assert checked_thread, "No samples found for the expected threads" +def test_max_time_usage(): + with pytest.raises(ValueError): + stack.StackCollector(None, max_time_usage_pct=0) + + +def test_max_time_usage_over(): + with pytest.raises(ValueError): + stack.StackCollector(None, max_time_usage_pct=200) + + @pytest.mark.parametrize( - ("stack_v2_enabled", "ignore_profiler"), [(True, True), (True, False), (False, True), (False, False)] + "stack_v2_enabled", + [True, False], +) +@pytest.mark.parametrize( + "ignore_profiler", + [True, False], ) def test_ignore_profiler(stack_v2_enabled, ignore_profiler, tmp_path): if sys.version_info[:2] == (3, 7) and stack_v2_enabled: @@ -691,3 +744,79 @@ def test_ignore_profiler(stack_v2_enabled, ignore_profiler, tmp_path): assert collector_worker_thread_id in thread_ids else: assert collector_worker_thread_id not in thread_ids + + +# TODO: support ignore profiler with stack_v2 and update this test +@pytest.mark.skipif(not TESTING_GEVENT, reason="Not testing gevent") +@pytest.mark.subprocess( + ddtrace_run=True, + env=dict(DD_PROFILING_IGNORE_PROFILER="1", DD_PROFILING_OUTPUT_PPROF="/tmp/test_ignore_profiler_gevent_task"), +) +def test_ignore_profiler_gevent_task(): + import gevent.monkey + + gevent.monkey.patch_all() + + import os + import time + import typing + + from ddtrace.profiling import collector + from ddtrace.profiling import event as event_mod + from ddtrace.profiling import profiler + from ddtrace.profiling.collector import stack + from tests.profiling.collector import pprof_utils + + def _fib(n): + if n == 1: + return 1 + elif n == 0: + return 0 + else: + return _fib(n - 1) + _fib(n - 2) + + class CollectorTest(collector.PeriodicCollector): + def collect(self) -> typing.Iterable[typing.Iterable[event_mod.Event]]: + _fib(22) + return [] + + output_filename = os.environ["DD_PROFILING_OUTPUT_PPROF"] + + p = profiler.Profiler() + + p.start() + + for c in p._profiler._collectors: + if isinstance(c, stack.StackCollector): + c.ignore_profiler + + c = CollectorTest(None, interval=0.00001) + c.start() + + time.sleep(3) + + worker_ident = c._worker.ident + + c.stop() + p.stop() + + profile = pprof_utils.parse_profile(output_filename + "." + str(os.getpid())) + + samples = pprof_utils.get_samples_with_value_type(profile, "cpu-time") + + thread_ids = set() + for sample in samples: + thread_id_label = pprof_utils.get_label_with_key(profile.string_table, sample, "thread id") + thread_id = int(thread_id_label.num) + thread_ids.add(thread_id) + + assert worker_ident not in thread_ids + + +def test_repr(): + test_collector._test_repr( + stack.StackCollector, + "StackCollector(status=, " + "recorder=Recorder(default_max_events=16384, max_events={}), min_interval_time=0.01, max_time_usage_pct=1.0, " + "nframes=64, ignore_profiler=False, endpoint_collection_enabled=None, tracer=None)", + ) diff --git a/tests/profiling_v2/simple_program.py b/tests/profiling_v2/simple_program.py new file mode 100755 index 00000000000..ed07bc5a402 --- /dev/null +++ b/tests/profiling_v2/simple_program.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +import os +import sys +import time + +from ddtrace.internal import service +from ddtrace.profiling import bootstrap +from ddtrace.profiling.collector import stack + + +for running_collector in bootstrap.profiler._profiler._collectors: + if isinstance(running_collector, stack.StackCollector): + break +else: + raise AssertionError("Unable to find stack collector") + + +print("hello world") +assert running_collector.status == service.ServiceStatus.RUNNING +print(running_collector.interval) + +t0 = time.time() +while time.time() - t0 < (running_collector.interval * 10): + pass + +# Do some serious memory allocations! +for _ in range(5000000): + object() + +print(os.getpid()) +print(bootstrap.profiler._profiler._stack_v2_enabled) +sys.exit(42) diff --git a/tests/profiling_v2/simple_program_fork.py b/tests/profiling_v2/simple_program_fork.py new file mode 100644 index 00000000000..ad8c0541ccd --- /dev/null +++ b/tests/profiling_v2/simple_program_fork.py @@ -0,0 +1,32 @@ +import os +import sys +import threading + +from ddtrace.internal import service +import ddtrace.profiling.auto +import ddtrace.profiling.bootstrap +import ddtrace.profiling.profiler + + +lock = threading.Lock() +lock.acquire() + + +assert ddtrace.profiling.bootstrap.profiler.status == service.ServiceStatus.RUNNING + + +child_pid = os.fork() +if child_pid == 0: + # Release it + lock.release() + + # We track this one though + lock = threading.Lock() + lock.acquire() + lock.release() +else: + lock.release() + assert ddtrace.profiling.bootstrap.profiler.status == service.ServiceStatus.RUNNING + print(child_pid) + pid, status = os.waitpid(child_pid, 0) + sys.exit(os.WEXITSTATUS(status)) diff --git a/tests/profiling_v2/simple_program_gevent.py b/tests/profiling_v2/simple_program_gevent.py new file mode 100644 index 00000000000..f50fa3aa2e0 --- /dev/null +++ b/tests/profiling_v2/simple_program_gevent.py @@ -0,0 +1,34 @@ +# Import from ddtrace before monkey patching to ensure that we grab all the +# necessary references to the unpatched modules. +import ddtrace.auto # noqa: F401, I001 +import ddtrace.profiling.auto # noqa:F401 + + +import gevent.monkey # noqa:F402 + +gevent.monkey.patch_all() + +import threading # noqa: E402, F402, I001 +import time # noqa: E402, F402 + + +def fibonacci(n): + if n == 0: + return 0 + elif n == 1: + return 1 + else: + return fibonacci(n - 1) + fibonacci(n - 2) + + +i = 1 +for _ in range(20): + threads = [] + for _ in range(10): + t = threading.Thread(target=fibonacci, args=(i,)) + t.start() + threads.append(t) + i += 1 + for t in threads: + t.join() + time.sleep(0.1) diff --git a/tests/profiling_v2/test_accuracy.py b/tests/profiling_v2/test_accuracy.py new file mode 100644 index 00000000000..61fbe3322ff --- /dev/null +++ b/tests/profiling_v2/test_accuracy.py @@ -0,0 +1,101 @@ +# -*- encoding: utf-8 -*- +import sys + +import pytest + + +@pytest.mark.subprocess( + env=dict(DD_PROFILING_MAX_TIME_USAGE_PCT="100", DD_PROFILING_OUTPUT_PPROF="/tmp/test_accuracy_libdd.pprof") +) +def test_accuracy_libdd(): + import collections + import os + + from ddtrace.profiling import profiler + from tests.profiling.collector import pprof_utils + from tests.profiling.test_accuracy import assert_almost_equal + from tests.profiling.test_accuracy import spend_16 + + # Set this to 100 so we don't sleep too often and mess with the precision. + p = profiler.Profiler() + p.start() + spend_16() + p.stop() + wall_times = collections.defaultdict(lambda: 0) + cpu_times = collections.defaultdict(lambda: 0) + profile = pprof_utils.parse_profile(os.environ["DD_PROFILING_OUTPUT_PPROF"] + "." + str(os.getpid())) + + for sample in profile.sample: + wall_time_index = pprof_utils.get_sample_type_index(profile, "wall-time") + + wall_time_spent_ns = sample.value[wall_time_index] + cpu_time_index = pprof_utils.get_sample_type_index(profile, "cpu-time") + cpu_time_spent_ns = sample.value[cpu_time_index] + + for location_id in sample.location_id: + location = pprof_utils.get_location_with_id(profile, location_id) + line = location.line[0] + function = pprof_utils.get_function_with_id(profile, line.function_id) + function_name = profile.string_table[function.name] + wall_times[function_name] += wall_time_spent_ns + cpu_times[function_name] += cpu_time_spent_ns + + assert_almost_equal(wall_times["spend_3"], 9e9) + assert_almost_equal(wall_times["spend_1"], 2e9) + assert_almost_equal(wall_times["spend_4"], 4e9) + assert_almost_equal(wall_times["spend_16"], 16e9) + assert_almost_equal(wall_times["spend_7"], 7e9) + + assert_almost_equal(wall_times["spend_cpu_2"], 2e9, tolerance=0.07) + assert_almost_equal(wall_times["spend_cpu_3"], 3e9, tolerance=0.07) + assert_almost_equal(cpu_times["spend_cpu_2"], 2e9, tolerance=0.07) + assert_almost_equal(cpu_times["spend_cpu_3"], 3e9, tolerance=0.07) + + +@pytest.mark.subprocess( + env=dict(DD_PROFILING_STACK_V2_ENABLED="1", DD_PROFILING_OUTPUT_PPROF="/tmp/test_accuracy_stack_v2.pprof") +) +@pytest.mark.skipif(sys.version_info[:2] == (3, 7), reason="stack_v2 is not supported on Python 3.7") +def test_accuracy_stack_v2(): + import collections + import os + + from ddtrace.profiling import profiler + from tests.profiling.collector import pprof_utils + from tests.profiling.test_accuracy import assert_almost_equal + from tests.profiling.test_accuracy import spend_16 + + # Set this to 100 so we don't sleep too often and mess with the precision. + p = profiler.Profiler() + p.start() + spend_16() + p.stop() + wall_times = collections.defaultdict(lambda: 0) + cpu_times = collections.defaultdict(lambda: 0) + profile = pprof_utils.parse_profile(os.environ["DD_PROFILING_OUTPUT_PPROF"] + "." + str(os.getpid())) + + for sample in profile.sample: + wall_time_index = pprof_utils.get_sample_type_index(profile, "wall-time") + + wall_time_spent_ns = sample.value[wall_time_index] + cpu_time_index = pprof_utils.get_sample_type_index(profile, "cpu-time") + cpu_time_spent_ns = sample.value[cpu_time_index] + + for location_id in sample.location_id: + location = pprof_utils.get_location_with_id(profile, location_id) + line = location.line[0] + function = pprof_utils.get_function_with_id(profile, line.function_id) + function_name = profile.string_table[function.name] + wall_times[function_name] += wall_time_spent_ns + cpu_times[function_name] += cpu_time_spent_ns + + assert_almost_equal(wall_times["spend_3"], 9e9) + assert_almost_equal(wall_times["spend_1"], 2e9) + assert_almost_equal(wall_times["spend_4"], 4e9) + assert_almost_equal(wall_times["spend_16"], 16e9) + assert_almost_equal(wall_times["spend_7"], 7e9) + + assert_almost_equal(wall_times["spend_cpu_2"], 2e9, tolerance=0.07) + assert_almost_equal(wall_times["spend_cpu_3"], 3e9, tolerance=0.07) + assert_almost_equal(cpu_times["spend_cpu_2"], 2e9, tolerance=0.07) + assert_almost_equal(cpu_times["spend_cpu_3"], 3e9, tolerance=0.07) diff --git a/tests/profiling_v2/test_main.py b/tests/profiling_v2/test_main.py new file mode 100644 index 00000000000..3142a1fbba8 --- /dev/null +++ b/tests/profiling_v2/test_main.py @@ -0,0 +1,227 @@ +# -*- encoding: utf-8 -*- +import multiprocessing +import os +import sys + +import pytest + +from tests.profiling.collector import lock_utils +from tests.profiling.collector import pprof_utils +from tests.utils import call_program +from tests.utils import flaky + + +@pytest.mark.parametrize("stack_v2_enabled", [True, False]) +def test_call_script(stack_v2_enabled): + if sys.version_info[:2] == (3, 7) and stack_v2_enabled: + pytest.skip("stack_v2 is not supported on Python 3.7") + env = os.environ.copy() + env["DD_PROFILING_ENABLED"] = "1" + env["DD_PROFILING_STACK_V2_ENABLED"] = "1" if stack_v2_enabled else "0" + stdout, stderr, exitcode, _ = call_program( + "ddtrace-run", sys.executable, os.path.join(os.path.dirname(__file__), "simple_program.py"), env=env + ) + if sys.platform == "win32": + assert exitcode == 0, (stdout, stderr) + else: + assert exitcode == 42, (stdout, stderr) + hello, interval, pid, stack_v2 = list(s.strip() for s in stdout.decode().strip().split("\n")) + assert hello == "hello world", stdout.decode().strip() + assert float(interval) >= 0.01, stdout.decode().strip() + assert stack_v2 == str(stack_v2_enabled) + + +@pytest.mark.skipif(not os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Not testing gevent") +@pytest.mark.parametrize("stack_v2_enabled", [True, False]) +def test_call_script_gevent(stack_v2_enabled): + if sys.version_info[:2] == (3, 7) and stack_v2_enabled: + pytest.skip("stack_v2 is not supported on Python 3.7") + if sys.version_info[:2] == (3, 8) and stack_v2_enabled: + pytest.skip("this test is flaky on 3.8 with stack v2") + env = os.environ.copy() + env["DD_PROFILING_ENABLED"] = "1" + env["DD_PROFILING_STACK_V2_ENABLED"] = "1" if stack_v2_enabled else "0" + stdout, stderr, exitcode, pid = call_program( + sys.executable, os.path.join(os.path.dirname(__file__), "simple_program_gevent.py"), env=env + ) + assert exitcode == 0, (stdout, stderr) + + +@pytest.mark.parametrize("stack_v2_enabled", [True, False]) +def test_call_script_pprof_output(stack_v2_enabled, tmp_path): + if sys.version_info[:2] == (3, 7) and stack_v2_enabled: + pytest.skip("stack_v2 is not supported on Python 3.7") + + """This checks if the pprof output and atexit register work correctly. + + The script does not run for one minute, so if the `stop_on_exit` flag is broken, this test will fail. + """ + filename = str(tmp_path / "pprof") + env = os.environ.copy() + env["DD_PROFILING_OUTPUT_PPROF"] = filename + env["DD_PROFILING_CAPTURE_PCT"] = "1" + env["DD_PROFILING_ENABLED"] = "1" + env["DD_PROFILING_STACK_V2_ENABLED"] = "1" if stack_v2_enabled else "0" + stdout, stderr, exitcode, _ = call_program( + "ddtrace-run", + sys.executable, + os.path.join(os.path.dirname(__file__), "../profiling", "simple_program.py"), + env=env, + ) + if sys.platform == "win32": + assert exitcode == 0, (stdout, stderr) + else: + assert exitcode == 42, (stdout, stderr) + _, _, _, pid = list(s.strip() for s in stdout.decode().strip().split("\n")) + profile = pprof_utils.parse_profile(filename + "." + str(pid)) + samples = pprof_utils.get_samples_with_value_type(profile, "cpu-time") + assert len(samples) > 0 + + +@pytest.mark.parametrize("stack_v2_enabled", [True, False]) +@pytest.mark.skipif(sys.platform == "win32", reason="fork only available on Unix") +def test_fork(stack_v2_enabled, tmp_path): + if sys.version_info[:2] == (3, 7) and stack_v2_enabled: + pytest.skip("stack_v2 is not supported on Python 3.7") + + filename = str(tmp_path / "pprof") + env = os.environ.copy() + env["DD_PROFILING_OUTPUT_PPROF"] = filename + env["DD_PROFILING_CAPTURE_PCT"] = "100" + env["DD_PROFILING_STACK_V2_ENABLED"] = "1" if stack_v2_enabled else "0" + stdout, stderr, exitcode, pid = call_program( + "python", os.path.join(os.path.dirname(__file__), "simple_program_fork.py"), env=env + ) + assert exitcode == 0 + child_pid = stdout.decode().strip() + profile = pprof_utils.parse_profile(filename + "." + str(pid)) + pprof_utils.assert_lock_events( + profile, + expected_acquire_events=[ + pprof_utils.LockAcquireEvent( + caller_name="", + filename="simple_program_fork.py", + linenos=lock_utils.LineNo(create=11, acquire=12, release=28), + lock_name="lock", + ), + ], + expected_release_events=[ + pprof_utils.LockReleaseEvent( + caller_name="", + filename="simple_program_fork.py", + linenos=lock_utils.LineNo(create=11, acquire=12, release=28), + lock_name="lock", + ), + ], + ) + child_profile = pprof_utils.parse_profile(filename + "." + str(child_pid)) + pprof_utils.assert_lock_events( + child_profile, + expected_acquire_events=[ + # After fork(), we clear the samples in child, so we only have one + # lock acquire event + pprof_utils.LockAcquireEvent( + caller_name="", + filename="simple_program_fork.py", + linenos=lock_utils.LineNo(create=24, acquire=25, release=26), + lock_name="lock", + ), + ], + expected_release_events=[ + pprof_utils.LockReleaseEvent( + caller_name="", + filename="simple_program_fork.py", + linenos=lock_utils.LineNo(create=11, acquire=12, release=21), + lock_name="lock", + ), + pprof_utils.LockReleaseEvent( + caller_name="", + filename="simple_program_fork.py", + linenos=lock_utils.LineNo(create=24, acquire=25, release=26), + lock_name="lock", + ), + ], + ) + + +@pytest.mark.parametrize("stack_v2_enabled", [True, False]) +@pytest.mark.skipif(sys.platform == "win32", reason="fork only available on Unix") +@pytest.mark.skipif(not os.getenv("DD_PROFILE_TEST_GEVENT", False), reason="Not testing gevent") +def test_fork_gevent(stack_v2_enabled): + if sys.version_info[:2] == (3, 7) and stack_v2_enabled: + pytest.skip("stack_v2 is not supported on Python 3.7") + env = os.environ.copy() + env["DD_PROFILING_STACK_V2_ENABLED"] = "1" if stack_v2_enabled else "0" + stdout, stderr, exitcode, pid = call_program( + "python", os.path.join(os.path.dirname(__file__), "../profiling", "gevent_fork.py"), env=env + ) + assert exitcode == 0 + + +methods = multiprocessing.get_all_start_methods() + + +@pytest.mark.parametrize("stack_v2_enabled", [True, False]) +@pytest.mark.parametrize( + "method", + set(methods) - {"forkserver", "fork"}, +) +def test_multiprocessing(stack_v2_enabled, method, tmp_path): + if sys.version_info[:2] == (3, 7) and stack_v2_enabled: + pytest.skip("stack_v2 is not supported on Python 3.7") + filename = str(tmp_path / "pprof") + env = os.environ.copy() + env["DD_PROFILING_OUTPUT_PPROF"] = filename + env["DD_PROFILING_ENABLED"] = "1" + env["DD_PROFILING_CAPTURE_PCT"] = "1" + env["DD_PROFILING_STACK_V2_ENABLED"] = "1" if stack_v2_enabled else "0" + stdout, stderr, exitcode, _ = call_program( + "ddtrace-run", + sys.executable, + os.path.join(os.path.dirname(__file__), "../profiling", "_test_multiprocessing.py"), + method, + env=env, + ) + assert exitcode == 0, (stdout, stderr) + pid, child_pid = list(s.strip() for s in stdout.decode().strip().split("\n")) + profile = pprof_utils.parse_profile(filename + "." + str(pid)) + samples = pprof_utils.get_samples_with_value_type(profile, "cpu-time") + assert len(samples) > 0 + child_profile = pprof_utils.parse_profile(filename + "." + str(child_pid)) + child_samples = pprof_utils.get_samples_with_value_type(child_profile, "cpu-time") + assert len(child_samples) > 0 + + +@flaky(1731959126) # Marking as flaky so it will show up in flaky reports +@pytest.mark.skipif(os.environ.get("GITLAB_CI") == "true", reason="Hanging and failing in GitLab CI") +@pytest.mark.subprocess( + ddtrace_run=True, + env=dict(DD_PROFILING_ENABLED="1"), + err=lambda _: "RuntimeError: the memalloc module is already started" not in _, +) +def test_memalloc_no_init_error_on_fork(): + import os + + pid = os.fork() + if not pid: + exit(0) + os.waitpid(pid, 0) + + +# Not parametrizing with stack_v2_enabled as subprocess mark doesn't support +# parametrized tests and this only tests our start up code. +@pytest.mark.subprocess( + ddtrace_run=True, + env=dict( + DD_PROFILING_ENABLED="1", + DD_UNLOAD_MODULES_FROM_SITECUSTOMIZE="1", + ), + out="OK\n", + err=None, +) +def test_profiler_start_up_with_module_clean_up_in_protobuf_app(): + # This can cause segfaults if we do module clean up with later versions of + # protobuf. This is a regression test. + from google.protobuf import empty_pb2 # noqa:F401 + + print("OK") diff --git a/tests/profiling_v2/test_profiler.py b/tests/profiling_v2/test_profiler.py new file mode 100644 index 00000000000..b5a2bb4bae8 --- /dev/null +++ b/tests/profiling_v2/test_profiler.py @@ -0,0 +1,204 @@ +import logging +import sys +import time + +import mock +import pytest + +import ddtrace +from ddtrace.profiling import collector +from ddtrace.profiling import exporter +from ddtrace.profiling import profiler +from ddtrace.profiling import scheduler +from ddtrace.profiling.collector import asyncio +from ddtrace.profiling.collector import stack +from ddtrace.profiling.collector import threading + + +def test_status(): + p = profiler.Profiler() + assert repr(p.status) == "" + p.start() + assert repr(p.status) == "" + p.stop(flush=False) + assert repr(p.status) == "" + + +def test_restart(): + p = profiler.Profiler() + p.start() + p.stop(flush=False) + p.start() + p.stop(flush=False) + + +def test_multiple_stop(): + """Check that the profiler can be stopped twice.""" + p = profiler.Profiler() + p.start() + p.stop(flush=False) + p.stop(flush=False) + + +def test_tracer_api(monkeypatch): + monkeypatch.setenv("DD_API_KEY", "foobar") + prof = profiler.Profiler(tracer=ddtrace.tracer) + assert prof.tracer == ddtrace.tracer + for col in prof._profiler._collectors: + if isinstance(col, stack.StackCollector): + assert col.tracer == ddtrace.tracer + break + else: + pytest.fail("Unable to find stack collector") + + +def test_profiler_init_float_division_regression(run_python_code_in_subprocess): + """ + Regression test for https://github.com/DataDog/dd-trace-py/pull/3751 + When float division is enabled, the value of `max_events` can be a `float`, + this is then passed as `deque(maxlen=float)` which is a type error + + File "/var/task/ddtrace/profiling/recorder.py", line 80, in _get_deque_for_event_type + return collections.deque(maxlen=self.max_events.get(event_type, self.default_max_events)) + TypeError: an integer is required + """ + code = """ +from ddtrace.profiling import profiler +from ddtrace.profiling.collector import stack_event + +prof = profiler.Profiler() + +# The error only happened for this specific kind of event +# DEV: Yes, this is likely a brittle way to test, but quickest/easiest way to trigger the error +prof._recorder.push_event(stack_event.StackExceptionSampleEvent()) + """ + + out, err, status, _ = run_python_code_in_subprocess(code) + assert status == 0, err + assert out == b"", err + assert err == b"" + + +@pytest.mark.subprocess() +def test_default_memory(): + from ddtrace.profiling import profiler + from ddtrace.profiling.collector import memalloc + + assert any(isinstance(col, memalloc.MemoryCollector) for col in profiler.Profiler()._profiler._collectors) + + +@pytest.mark.subprocess(env=dict(DD_PROFILING_MEMORY_ENABLED="true")) +def test_enable_memory(): + from ddtrace.profiling import profiler + from ddtrace.profiling.collector import memalloc + + assert any(isinstance(col, memalloc.MemoryCollector) for col in profiler.Profiler()._profiler._collectors) + + +@pytest.mark.subprocess(env=dict(DD_PROFILING_MEMORY_ENABLED="false")) +def test_disable_memory(): + from ddtrace.profiling import profiler + from ddtrace.profiling.collector import memalloc + + assert all(not isinstance(col, memalloc.MemoryCollector) for col in profiler.Profiler()._profiler._collectors) + + +def test_copy(): + p = profiler._ProfilerInstance(env="123", version="dwq", service="foobar") + c = p.copy() + assert c == p + assert p.env == c.env + assert p.version == c.version + assert p.service == c.service + assert p.tracer == c.tracer + assert p.tags == c.tags + + +def test_failed_start_collector(caplog, monkeypatch): + class ErrCollect(collector.Collector): + def _start_service(self): + raise RuntimeError("could not import required module") + + def _stop_service(self): + pass + + @staticmethod + def collect(): + pass + + @staticmethod + def snapshot(): + raise Exception("error!") + + monkeypatch.setenv("DD_PROFILING_UPLOAD_INTERVAL", "1") + + class Exporter(exporter.Exporter): + def export(self, events, *args, **kwargs): + pass + + class TestProfiler(profiler._ProfilerInstance): + def _build_default_exporters(self, *args, **kargs): + return [Exporter()] + + p = TestProfiler() + err_collector = mock.MagicMock(wraps=ErrCollect(p._recorder)) + p._collectors = [err_collector] + p.start() + + def profiling_tuples(tuples): + return [t for t in tuples if t[0].startswith("ddtrace.profiling")] + + assert profiling_tuples(caplog.record_tuples) == [ + ("ddtrace.profiling.profiler", logging.ERROR, "Failed to start collector %r, disabling." % err_collector) + ] + time.sleep(2) + p.stop() + assert err_collector.snapshot.call_count == 0 + assert profiling_tuples(caplog.record_tuples) == [ + ("ddtrace.profiling.profiler", logging.ERROR, "Failed to start collector %r, disabling." % err_collector) + ] + + +def test_default_collectors(): + p = profiler.Profiler() + assert any(isinstance(c, stack.StackCollector) for c in p._profiler._collectors) + assert any(isinstance(c, threading.ThreadingLockCollector) for c in p._profiler._collectors) + try: + import asyncio as _ # noqa: F401 + except ImportError: + pass + else: + assert any(isinstance(c, asyncio.AsyncioLockCollector) for c in p._profiler._collectors) + p.stop(flush=False) + + +def test_profiler_serverless(monkeypatch): + # type: (...) -> None + monkeypatch.setenv("AWS_LAMBDA_FUNCTION_NAME", "foobar") + p = profiler.Profiler() + assert isinstance(p._scheduler, scheduler.ServerlessScheduler) + assert p.tags["functionname"] == "foobar" + + +@pytest.mark.skipif(sys.version_info < (3, 8), reason="Python 3.7 deprecation warning") +@pytest.mark.subprocess() +def test_profiler_ddtrace_deprecation(): + """ + ddtrace interfaces loaded by the profiler can be marked deprecated, and we should update + them when this happens. As reported by https://github.com/DataDog/dd-trace-py/issues/8881 + """ + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + from ddtrace.profiling import _threading # noqa:F401 + from ddtrace.profiling import event # noqa:F401 + from ddtrace.profiling import profiler # noqa:F401 + from ddtrace.profiling import recorder # noqa:F401 + from ddtrace.profiling import scheduler # noqa:F401 + from ddtrace.profiling.collector import _lock # noqa:F401 + from ddtrace.profiling.collector import _task # noqa:F401 + from ddtrace.profiling.collector import _traceback # noqa:F401 + from ddtrace.profiling.collector import memalloc # noqa:F401 + from ddtrace.profiling.collector import stack # noqa:F401 + from ddtrace.profiling.collector import stack_event # noqa:F401 diff --git a/tests/profiling_v2/test_scheduler.py b/tests/profiling_v2/test_scheduler.py new file mode 100644 index 00000000000..dc3c2c0d7d1 --- /dev/null +++ b/tests/profiling_v2/test_scheduler.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +import logging +import time + +import mock + +from ddtrace.profiling import exporter +from ddtrace.profiling import scheduler + + +def test_thread_name(): + exp = exporter.NullExporter() + s = scheduler.Scheduler(None, [exp]) + s.start() + assert s._worker.name == "ddtrace.profiling.scheduler:Scheduler" + s.stop() + + +def test_before_flush(): + x = {} + + def call_me(): + x["OK"] = True + + s = scheduler.Scheduler(None, [exporter.NullExporter()], before_flush=call_me) + s.flush() + assert x["OK"] + + +def test_before_flush_failure(caplog): + def call_me(): + raise Exception("LOL") + + s = scheduler.Scheduler(None, [exporter.NullExporter()], before_flush=call_me) + s.flush() + assert caplog.record_tuples == [ + (("ddtrace.profiling.scheduler", logging.ERROR, "Scheduler before_flush hook failed")) + ] + + +@mock.patch("ddtrace.profiling.scheduler.Scheduler.periodic") +def test_serverless_periodic(mock_periodic): + s = scheduler.ServerlessScheduler(None, [exporter.NullExporter()]) + # Fake start() + s._last_export = time.time_ns() + s.periodic() + assert s._profiled_intervals == 1 + mock_periodic.assert_not_called() + s._last_export = time.time_ns() - 65 + s._profiled_intervals = 65 + s.periodic() + assert s._profiled_intervals == 0 + assert s.interval == 1 + mock_periodic.assert_called() diff --git a/tests/smoke_test.py b/tests/smoke_test.py index cbf5ebc8e61..24017c0df81 100644 --- a/tests/smoke_test.py +++ b/tests/smoke_test.py @@ -37,14 +37,8 @@ def emit(self, record): try: from ddtrace.appsec._iast._taint_tracking._native import ops - if os.environ.get("DD_IAST_ENABLED") == "False": - assert any( - "IAST not enabled but native module is being loaded" in message - for message in log_messages - ) - else: - assert ops - assert len(log_messages) == 0 + assert ops + assert len(log_messages) == 0 except ImportError as e: assert False, "Importing the native module failed, _native probably not compiled correctly: %s" % str(e) """ diff --git a/tests/submod/stuff.py b/tests/submod/stuff.py index d2fa07e0ec5..375520bf76b 100644 --- a/tests/submod/stuff.py +++ b/tests/submod/stuff.py @@ -125,7 +125,7 @@ def __init__(self): foo = property(operator.attrgetter("_foo")) -from ddtrace.internal.compat import monotonic_ns # noqa:E402 +from time import monotonic_ns # noqa:E402 def durationstuff(ns): diff --git a/tests/tracer/test_rate_limiter.py b/tests/tracer/test_rate_limiter.py index d66f980cbc3..df9e1458525 100644 --- a/tests/tracer/test_rate_limiter.py +++ b/tests/tracer/test_rate_limiter.py @@ -1,9 +1,10 @@ from __future__ import division +import time + import mock import pytest -from ddtrace.internal import compat from ddtrace.internal.rate_limiter import BudgetRateLimiterWithJitter from ddtrace.internal.rate_limiter import RateLimiter from ddtrace.internal.rate_limiter import RateLimitExceeded @@ -20,7 +21,7 @@ def test_rate_limiter_init(time_window): assert limiter.rate_limit == 100 assert limiter.tokens == 100 assert limiter.max_tokens == 100 - assert limiter.last_update_ns <= compat.monotonic_ns() + assert limiter.last_update_ns <= time.monotonic_ns() @pytest.mark.parametrize("time_window", [1e3, 1e6, 1e9]) @@ -30,10 +31,10 @@ def test_rate_limiter_rate_limit_0(time_window): assert limiter.tokens == 0 assert limiter.max_tokens == 0 - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() for i in nanoseconds(10000, time_window): # Make sure the time is different for every check - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns + i): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns + i): assert limiter.is_allowed() is False @@ -44,10 +45,10 @@ def test_rate_limiter_rate_limit_negative(time_window): assert limiter.tokens == -1 assert limiter.max_tokens == -1 - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() for i in nanoseconds(10000, time_window): # Make sure the time is different for every check - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns + i): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns + i): assert limiter.is_allowed() is True @@ -66,12 +67,12 @@ def check_limit(): assert limiter.is_allowed() is False # Start time - now = compat.monotonic_ns() + now = time.monotonic_ns() # Check the limit for 5 time frames for i in nanoseconds(5, time_window): # Keep the same timeframe - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now + i): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now + i): check_limit() @@ -80,14 +81,14 @@ def test_rate_limiter_is_allowed_large_gap(time_window): limiter = RateLimiter(rate_limit=100, time_window=time_window) # Start time - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() # Keep the same timeframe - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns): for _ in range(100): assert limiter.is_allowed() is True # Large gap before next call to `is_allowed()` - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns + (time_window * 100)): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns + (time_window * 100)): for _ in range(100): assert limiter.is_allowed() is True @@ -97,13 +98,13 @@ def test_rate_limiter_is_allowed_small_gaps(time_window): limiter = RateLimiter(rate_limit=100, time_window=time_window) # Start time - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() gap = 1e9 / 100 # Keep incrementing by a gap to keep us at our rate limit for i in nanoseconds(10000, time_window): # Keep the same timeframe time_ns = now_ns + (gap * i) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is True @@ -112,8 +113,8 @@ def test_rate_liimter_effective_rate_rates(time_window): limiter = RateLimiter(rate_limit=100, time_window=time_window) # Static rate limit window - starting_window_ns = compat.monotonic_ns() - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=starting_window_ns): + starting_window_ns = time.monotonic_ns() + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=starting_window_ns): for _ in range(100): assert limiter.is_allowed() is True assert limiter.effective_rate == 1.0 @@ -127,7 +128,7 @@ def test_rate_liimter_effective_rate_rates(time_window): prev_rate = 0.5 window_ns = starting_window_ns + time_window - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=window_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=window_ns): for _ in range(100): assert limiter.is_allowed() is True assert limiter.effective_rate == 0.75 @@ -144,7 +145,7 @@ def test_rate_liimter_effective_rate_rates(time_window): def test_rate_limiter_effective_rate_starting_rate(time_window): limiter = RateLimiter(rate_limit=1, time_window=time_window) - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() # Default values assert limiter.current_window_ns == 0 @@ -156,7 +157,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): assert limiter.prev_window_rate is None # Calling `.is_allowed()` updates the values - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns): assert limiter.is_allowed() is True assert limiter.effective_rate == 1.0 assert limiter.current_window_ns == now_ns @@ -164,7 +165,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Gap of 0.85 seconds, same window time_ns = now_ns + (0.85 * time_window) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is False # DEV: We have rate_limit=1 set assert limiter.effective_rate == 0.5 @@ -173,7 +174,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Gap of 1.0 seconds, new window time_ns = now_ns + time_window - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is True assert limiter.effective_rate == 0.75 assert limiter.current_window_ns == (now_ns + time_window) @@ -181,7 +182,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Gap of 1.85 seconds, same window time_ns = now_ns + (1.85 * time_window) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is False assert limiter.effective_rate == 0.5 assert limiter.current_window_ns == (now_ns + time_window) # Same as old window @@ -189,7 +190,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Large gap of 100 seconds, new window time_ns = now_ns + (100.0 * time_window) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is True assert limiter.effective_rate == 0.75 assert limiter.current_window_ns == (now_ns + (100.0 * time_window))