Fix ignore_keys issue in detailed__dict__

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 8.5.0
+current_version = 8.6.1
 commit = True
 tag = True
 tag_name = {new_version}

AUTHORS.md

@@ -75,3 +75,4 @@ Authors in order of the timeline of their contributions:
 - [dtorres-sf](https://github.com/dtorres-sf) for the fix for moving nested tables when using iterable_compare_func.
 - [Jim Cipar](https://github.com/jcipar) for the fix recursion depth limit when hashing numpy.datetime64
 - [Enji Cooper](https://github.com/ngie-eign) for converting legacy setuptools use to pyproject.toml
+- [Diogo Correia](https://github.com/diogotcorreia) for reporting security vulnerability in Delta and DeepDiff that could allow remote code execution.

CHANGELOG.md

@@ -1,5 +1,9 @@
 # DeepDiff Change log
 
+- v8-6-1
+    - Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization).
+
+
 - v8-6-0
     - Added Colored View thanks to @mauvilsa 
     - Added support for applying deltas to NamedTuple thanks to @paulsc 

CITATION.cff

@@ -5,6 +5,6 @@ authors:
   given-names: "Sep"
   orcid: "https://orcid.org/0009-0009-5828-4345"
 title: "DeepDiff"
-version: 8.5.0
+version: 8.6.1
 date-released: 2024
 url: "https://github.com/seperman/deepdiff"

README.md

@@ -1,4 +1,4 @@
-# DeepDiff v 8.5.0
+# DeepDiff v 8.6.1
 
 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat)
 ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat)
@@ -17,12 +17,15 @@
 
 Tested on Python 3.9+ and PyPy3.
 
-- **[Documentation](https://zepworks.com/deepdiff/8.5.0/)**
+- **[Documentation](https://zepworks.com/deepdiff/8.6.1/)**
 
 ## What is new?
 
 Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
+DeepDiff 8-6-1
+- Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization).
+
 DeepDiff 8-6-0
 
 - Added Colored View thanks to @mauvilsa 

deepdiff/__init__.py

@@ -1,6 +1,6 @@
 """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes."""
 # flake8: noqa
-__version__ = '8.5.0'
+__version__ = '8.6.1'
 import logging
 
 if __name__ == '__main__':

deepdiff/delta.py

@@ -17,7 +17,7 @@
 )
 from deepdiff.path import (
     _path_to_elements, _get_nested_obj, _get_nested_obj_and_force,
-    GET, GETATTR, parse_path, stringify_path,
+    GET, GETATTR, check_elem, parse_path, stringify_path,
 )
 from deepdiff.anyset import AnySet
 from deepdiff.summarize import summarize
@@ -237,6 +237,11 @@ def _get_elem_and_compare_to_old_value(
         forced_old_value=None,
         next_element=None,
     ):
+        try:
+            check_elem(elem)
+        except ValueError as error:
+            self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path_for_err_reporting, error))
+            return not_found
         # if forced_old_value is not None:
         try:
             if action == GET:
@@ -536,6 +541,7 @@ def _get_elements_and_details(self, path):
                 obj = self
                 # obj = self.get_nested_obj(obj=self, elements=elements[:-1])
             elem, action = elements[-1]  # type: ignore
+            check_elem(elem)
         except Exception as e:
             self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path, e))
             return None

deepdiff/helper.py

@@ -739,8 +739,8 @@ def detailed__dict__(obj: Any, ignore_private_variables: bool = True, ignore_key
     else:
         result = obj.__dict__.copy()  # A shallow copy
         private_var_prefix = f"_{obj.__class__.__name__}__"  # The semi private variables in Python get this prefix
-        for key in ignore_keys:
-            if key in result or (
+        for key in obj.__dict__:
+            if key in ignore_keys or (
                 ignore_private_variables and key.startswith('__') and not key.startswith(private_var_prefix)
             ):
                 del result[key]

deepdiff/path.py

@@ -117,6 +117,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
 
 def _get_nested_obj(obj, elements, next_element=None):
     for (elem, action) in elements:
+        check_elem(elem)
         if action == GET:
             obj = obj[elem]
         elif action == GETATTR:
@@ -134,11 +135,17 @@ def _guess_type(elements, elem, index, next_element):
     return {}
 
 
+def check_elem(elem):
+    if isinstance(elem, str) and elem.startswith("__") and elem.endswith("__"):
+        raise ValueError("traversing dunder attributes is not allowed")
+
+
 def _get_nested_obj_and_force(obj, elements, next_element=None):
     prev_elem = None
     prev_action = None
     prev_obj = obj
     for index, (elem, action) in enumerate(elements):
+        check_elem(elem)
         _prev_obj = obj
         if action == GET:
             try:

deepdiff/serialization.py

@@ -59,7 +59,7 @@ class UnsupportedFormatErr(TypeError):
 DELTA_IGNORE_ORDER_NEEDS_REPETITION_REPORT = 'report_repetition must be set to True when ignore_order is True to create the delta object.'
 DELTA_ERROR_WHEN_GROUP_BY = 'Delta can not be made when group_by is used since the structure of data is modified from the original form.'
 
-SAFE_TO_IMPORT = {
+SAFE_TO_IMPORT = frozenset({
     'builtins.range',
     'builtins.complex',
     'builtins.set',
@@ -95,7 +95,7 @@ class UnsupportedFormatErr(TypeError):
     'ipaddress.IPv4Address',
     'ipaddress.IPv6Address',
     'collections.abc.KeysView',
-}
+})
 
 
 TYPE_STR_TO_TYPE = {

docs/authors.rst

@@ -117,6 +117,7 @@ and polars support.
   limit when hashing numpy.datetime64
 - `Enji Cooper <https://github.com/ngie-eign>`__ for converting legacy
   setuptools use to pyproject.toml
+- `Diogo Correia <https://github.com/diogotcorreia>`__ for reporting security vulnerability in Delta and DeepDiff that could allow remote code execution.
 
 
 .. _Sep Dehpour (Seperman): http://www.zepworks.com

docs/changelog.rst

@@ -5,6 +5,9 @@ Changelog
 
 DeepDiff Changelog
 
+- v8-6-1
+   - Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization).
+
 - v8-6-0
    - Added Colored View thanks to @mauvilsa
    - Added support for applying deltas to NamedTuple thanks to @paulsc

docs/conf.py

@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '8.5.0'
+version = '8.6.1'
 # The full version, including alpha/beta/rc tags.
-release = '8.5.0'
+release = '8.6.1'
 
 load_dotenv(override=True)
 DOC_VERSION = os.environ.get('DOC_VERSION', version)

docs/index.rst

@@ -4,7 +4,7 @@
    contain the root `toctree` directive.
 
 
-DeepDiff 8.5.0 documentation!
+DeepDiff 8.6.1 documentation!
 =============================
 
 *******
@@ -31,6 +31,40 @@ The DeepDiff library includes the following modules:
 What Is New
 ***********
 
+DeepDiff 8-6-1
+--------------
+
+    - Patched security vulnerability in the Delta class which was vulnerable to class pollution via its constructor, and when combined with a gadget available in DeltaDiff itself, it could lead to Denial of Service and Remote Code Execution (via insecure Pickle deserialization).
+
+
+DeepDiff 8-6-0
+--------------
+
+   - Added Colored View thanks to @mauvilsa
+   - Added support for applying deltas to NamedTuple thanks to @paulsc
+   - Fixed test_delta.py with Python 3.14 thanks to @Romain-Geissler-1A
+   - Added python property serialization to json
+   - Added ip address serialization
+   - Switched to UV from pip
+   - Added Claude.md
+   - Added uuid hashing thanks to @akshat62
+   - Added ``ignore_uuid_types`` flag to DeepDiff to avoid type reports
+     when comparing UUID and string.
+   - Added comprehensive type hints across the codebase (multiple commits
+     for better type safety)
+   - Added support for memoryview serialization
+   - Added support for bytes serialization (non-UTF8 compatible)
+   - Fixed bug where group_by with numbers would leak type info into group
+     path reports
+   - Fixed bug in ``_get_clean_to_keys_mapping without`` explicit
+     significant digits
+   - Added support for python dict key serialization
+   - Enhanced support for IP address serialization with safe module imports
+   - Added development tooling improvements (pyright config, .envrc
+     example)
+   - Updated documentation and development instructions
+
+
 DeepDiff 8-5-0
 --------------
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "deepdiff"
-version = "8.5.0"
+version = "8.6.1"
 dependencies = [
   "orderly-set>=5.4.1,<6",
 ]

tests/test_security.py

@@ -0,0 +1,133 @@
+import os
+import pickle
+import pytest
+from deepdiff import Delta
+from deepdiff.helper import Opcode
+from deepdiff.serialization import ForbiddenModule
+
+
+class TestDeltaClassPollution:
+
+    def test_builtins_int(self):
+
+        pollute_int = pickle.dumps(
+            {
+                "values_changed": {"root['tmp']": {"new_value": Opcode("", 0, 0, 0, 0)}},
+                "dictionary_item_added": {
+                    (
+                        ("root", "GETATTR"),
+                        ("tmp", "GET"),
+                        ("__repr__", "GETATTR"),
+                        ("__globals__", "GETATTR"),
+                        ("__builtins__", "GET"),
+                        ("int", "GET"),
+                    ): "no longer a class"
+                },
+            }
+        )
+
+        assert isinstance(pollute_int, bytes)
+
+        # ------------[ Exploit ]------------
+        # This could be some example, vulnerable, application.
+        # The inputs above could be sent via HTTP, for example.
+
+
+        # Existing dictionary; it is assumed that it contains
+        # at least one entry, otherwise a different Delta needs to be
+        # applied first, adding an entry to the dictionary.
+        mydict = {"tmp": "foobar"}
+
+        # Before pollution
+        assert 42 == int("41") + 1
+
+        # Apply Delta to mydict
+        result = mydict + Delta(pollute_int)
+
+        assert 1337 == int("1337")
+
+    def test_remote_code_execution(self):
+        if os.path.exists('/tmp/pwned'):
+            os.remove('/tmp/pwned')
+
+        pollute_safe_to_import = pickle.dumps(
+            {
+                "values_changed": {"root['tmp']": {"new_value": Opcode("", 0, 0, 0, 0)}},
+                "set_item_added": {
+                    (
+                        ("root", "GETATTR"),
+                        ("tmp", "GET"),
+                        ("__repr__", "GETATTR"),
+                        ("__globals__", "GETATTR"),
+                        ("sys", "GET"),
+                        ("modules", "GETATTR"),
+                        ("deepdiff.serialization", "GET"),
+                        ("SAFE_TO_IMPORT", "GETATTR"),
+                    ): set(["posix.system"])
+                },
+            }
+        )
+
+        # From https://davidhamann.de/2020/04/05/exploiting-python-pickle/
+        class RCE:
+            def __reduce__(self):
+                cmd = "id > /tmp/pwned"
+                return os.system, (cmd,)
+
+        # Wrap object with dictionary so that Delta does not crash
+        rce_pickle = pickle.dumps({"_": RCE()})
+
+        assert isinstance(pollute_safe_to_import, bytes)
+        assert isinstance(rce_pickle, bytes)
+
+        # ------------[ Exploit ]------------
+        # This could be some example, vulnerable, application.
+        # The inputs above could be sent via HTTP, for example.
+
+        # Existing dictionary; it is assumed that it contains
+        # at least one entry, otherwise a different Delta needs to be
+        # applied first, adding an entry to the dictionary.
+        mydict = {"tmp": "foobar"}
+
+        # Apply Delta to mydict
+        with pytest.raises(ValueError) as exc_info:
+            mydict + Delta(pollute_safe_to_import)
+        assert "traversing dunder attributes is not allowed" == str(exc_info.value)
+
+        with pytest.raises(ForbiddenModule) as exc_info:
+            Delta(rce_pickle)  # no need to apply this Delta
+        assert "Module 'posix.system' is forbidden. You need to explicitly pass it by passing a safe_to_import parameter" == str(exc_info.value)
+
+        assert not os.path.exists('/tmp/pwned'), "We should not have created this file"
+
+    def test_delta_should_not_access_globals(self):
+
+        pollute_global = pickle.dumps(
+            {
+                "dictionary_item_added": {
+                    (
+                        ("root", "GETATTR"),
+                        ("myfunc", "GETATTR"),
+                        ("__globals__", "GETATTR"),
+                        ("PWNED", "GET"),
+                    ): 1337
+                }
+            }
+        )
+
+
+        # demo application
+        class Foo:
+            def __init__(self):
+                pass
+
+            def myfunc(self):
+                pass
+
+
+        PWNED = False
+        delta = Delta(pollute_global)
+        assert PWNED is False
+        b = Foo() + delta
+
+        assert PWNED is False