From 56859dee2aa62efbe19419735de427d3dcf6a7aa Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 19:45:07 +0300 Subject: [PATCH 01/23] First implementation of OOP interface (DDict class) --- dpath/__init__.py | 1 + dpath/ddict.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 dpath/ddict.py diff --git a/dpath/__init__.py b/dpath/__init__.py index e69de29..5eac182 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -0,0 +1 @@ +from dpath.ddict import DDict diff --git a/dpath/ddict.py b/dpath/ddict.py new file mode 100644 index 0000000..5097838 --- /dev/null +++ b/dpath/ddict.py @@ -0,0 +1,43 @@ +from typing import Callable, Any, List, Dict + +from dpath import util +from dpath.util import MERGE_ADDITIVE + +FilterType = Callable[[Any], bool] # (Any) -> bool + +_DEFAULT_SENTINEL: Any = object() + + +class DDict(dict): + """ + Glob aware dict + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def set(self, glob: str, value, separator="/", afilter: FilterType = None): + return util.set(self, glob, value, separator=separator, afilter=afilter) + + def get(self, glob: str, separator="/", default=_DEFAULT_SENTINEL) -> Any: + """ + Same as dict.get but glob aware + """ + if default is not _DEFAULT_SENTINEL: + # Default value was passed + return util.get(self, glob, separator=separator, default=default) + else: + # Let util.get handle default value + return util.get(self, glob, separator=separator) + + def values(self, glob="*", separator="/", afilter: FilterType = None, dirs=True) -> List: + """ + Same as dict.values but glob aware + """ + return util.values(self, glob, separator=separator, afilter=afilter, dirs=dirs) + + def search(self, glob, yielded=False, separator="/", afilter: FilterType = None, dirs=True): + return util.search(self, glob, yielded=yielded, separator=separator, afilter=afilter, dirs=dirs) + + def merge(self, src: Dict, separator="/", afilter: FilterType = None, flags=MERGE_ADDITIVE): + return util.merge(self, src, separator=separator, afilter=afilter, flags=flags) From 5c9749591b953a88fc91a0de60231172235d8257 Mon Sep 17 00:00:00 2001 From: moomoohk Date: Sun, 12 Sep 2021 20:01:04 +0300 Subject: [PATCH 02/23] Explicitly set public exports --- dpath/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dpath/__init__.py b/dpath/__init__.py index 5eac182..87f4114 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -1 +1,5 @@ from dpath.ddict import DDict + +__all__ = [ + DDict, +] From 687496c1f70578a774164d7396a8b767db7b3d62 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 16:53:16 +0200 Subject: [PATCH 03/23] Merge branch 'master' into feature/oop-support --- .github/workflows/deploy.yml | 1 + .github/workflows/tests.yml | 33 +- MAINTAINERS.md | 2 +- README.rst | 250 ++++++------ dpath/__init__.py | 358 ++++++++++++++++- dpath/ddict.py | 29 +- dpath/segments.py | 170 ++++---- dpath/types.py | 45 +++ dpath/util.py | 367 ++---------------- dpath/version.py | 2 +- flake8.ini | 5 + setup.py | 18 +- tests/__init__.py | 3 + tests/test_broken_afilter.py | 26 +- tests/{test_util_delete.py => test_delete.py} | 25 +- ..._util_get_values.py => test_get_values.py} | 101 ++--- tests/{test_util_merge.py => test_merge.py} | 71 ++-- tests/{test_util_new.py => test_new.py} | 56 +-- tests/test_path_get.py | 6 +- tests/test_path_paths.py | 11 +- tests/test_paths.py | 9 + tests/test_search.py | 238 ++++++++++++ tests/test_segments.py | 364 +++++++++-------- tests/test_set.py | 91 +++++ tests/test_types.py | 60 +-- tests/test_unicode.py | 30 +- tests/test_util_paths.py | 9 - tests/test_util_search.py | 238 ------------ tests/test_util_set.py | 91 ----- tox.ini | 15 +- 30 files changed, 1442 insertions(+), 1282 deletions(-) create mode 100644 dpath/types.py create mode 100644 flake8.ini create mode 100644 tests/__init__.py rename tests/{test_util_delete.py => test_delete.py} (54%) rename tests/{test_util_get_values.py => test_get_values.py} (54%) rename tests/{test_util_merge.py => test_merge.py} (59%) rename tests/{test_util_new.py => test_new.py} (51%) create mode 100644 tests/test_paths.py create mode 100644 tests/test_search.py create mode 100644 tests/test_set.py delete mode 100644 tests/test_util_paths.py delete mode 100644 tests/test_util_search.py delete mode 100644 tests/test_util_set.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8210fac..5c1386e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -46,6 +46,7 @@ jobs: uses: loopwerk/tag-changelog@v1 with: token: ${{ secrets.GITHUB_TOKEN }} + config_file: .github/tag-changelog-config.js - name: PyPI Deployment uses: casperdcl/deploy-pypi@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 328cb2a..ba8e2bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,6 +21,28 @@ on: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: + + # Run flake8 linter + flake8: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@main + + - name: Set up Python 3.10 + uses: actions/setup-python@main + with: + python-version: "3.10" + + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1.1 + + - name: Lint with flake8 + run: | + pip install flake8 + flake8 setup.py dpath/ tests/ + # Generate a common hashseed for all tests generate-hashseed: runs-on: ubuntu-latest @@ -32,28 +54,29 @@ jobs: - name: Generate Hashseed id: generate run: | - python -c "from random import randint; + python -c "import os + from random import randint hashseed = randint(0, 4294967295) print(f'{hashseed=}') - print(f'::set-output name=hashseed::{hashseed}')" + open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" # Tests job tests: # The type of runner that the job will run on runs-on: ubuntu-latest - needs: generate-hashseed + needs: [generate-hashseed, flake8] strategy: matrix: # Match versions specified in tox.ini - python-version: [3.6, 3.8, 3.9, pypy-3.7] + python-version: ['3.8', '3.9', '3.10', 'pypy-3.7'] # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@main - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@main diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 0f9fb86..ee327f6 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -10,7 +10,7 @@ There are several individuals in the community who have taken an active role in Where and How do we communicate =============================== -The dpath maintainers communcate in 3 primary ways: +The dpath maintainers communicate in 3 primary ways: 1. Email, directly to each other. 2. Github via issue and pull request comments diff --git a/README.rst b/README.rst index ea88f5a..0ad3ad2 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,7 @@ dpath-python ============ |PyPI| +|Python Version| |Build Status| |Gitter| @@ -30,7 +31,7 @@ Using Dpath .. code-block:: python - import dpath.util + import dpath Separators ========== @@ -49,10 +50,10 @@ Suppose we have a dictionary like this: x = { "a": { "b": { - "3": 2, - "43": 30, - "c": [], - "d": ['red', 'buggy', 'bumpers'], + "3": 2, + "43": 30, + "c": [], + "d": ['red', 'buggy', 'bumpers'], } } } @@ -62,8 +63,8 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. .. code-block:: pycon - >>> help(dpath.util.get) - Help on function get in module dpath.util: + >>> help(dpath.get) + Help on function get in module dpath: get(obj, glob, separator='/') Given an object which contains only one possible match for the given glob, @@ -72,7 +73,7 @@ key '43' in the 'b' hash which is in the 'a' hash". That's easy. If more than one leaf matches the glob, ValueError is raised. If the glob is not found, KeyError is raised. - >>> dpath.util.get(x, '/a/b/43') + >>> dpath.get(x, '/a/b/43') 30 Or you could say "Give me a new dictionary with the values of all @@ -80,8 +81,8 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> help(dpath.util.search) - Help on function search in module dpath.util: + >>> help(dpath.search) + Help on function search in module dpath: search(obj, glob, yielded=False) Given a path glob, return a dictionary containing all keys @@ -95,27 +96,27 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. .. code-block:: pycon - >>> result = dpath.util.search(x, "a/b/[cd]") - >>> print json.dumps(result, indent=4, sort_keys=True) + >>> result = dpath.search(x, "a/b/[cd]") + >>> print(json.dumps(result, indent=4, sort_keys=True)) { - "a": { - "b": { - "c": [], - "d": [ - "red", - "buggy", - "bumpers" - ] + "a": { + "b": { + "c": [], + "d": [ + "red", + "buggy", + "bumpers" + ] + } } } - } ... Wow that was easy. What if I want to iterate over the results, and not get a merged view? .. code-block:: pycon - >>> for x in dpath.util.search(x, "a/b/[cd]", yielded=True): print x + >>> for x in dpath.search(x, "a/b/[cd]", yielded=True): print(x) ... ('a/b/c', []) ('a/b/d', ['red', 'buggy', 'bumpers']) @@ -125,8 +126,8 @@ don't care about the paths they were found at: .. code-block:: pycon - >>> help(dpath.util.values) - Help on function values in module dpath.util: + >>> help(dpath.values) + Help on function values in module dpath: values(obj, glob, separator='/', afilter=None, dirs=True) Given an object and a path glob, return an array of all values which match @@ -134,7 +135,7 @@ don't care about the paths they were found at: and it is primarily a shorthand for a list comprehension over a yielded search call. - >>> dpath.util.values(x, '/a/b/d/*') + >>> dpath.values(x, '/a/b/d/*') ['red', 'buggy', 'bumpers'] Example: Setting existing keys @@ -145,16 +146,16 @@ value 'Waffles'. .. code-block:: pycon - >>> help(dpath.util.set) - Help on function set in module dpath.util: + >>> help(dpath.set) + Help on function set in module dpath: set(obj, glob, value) Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. - >>> dpath.util.set(x, 'a/b/[cd]', 'Waffles') + >>> dpath.set(x, 'a/b/[cd]', 'Waffles') 2 - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -175,8 +176,8 @@ necessary to get to the terminus. .. code-block:: pycon - >>> help(dpath.util.new) - Help on function new in module dpath.util: + >>> help(dpath.new) + Help on function new in module dpath: new(obj, path, value) Set the element at the terminus of path to value, and create @@ -187,8 +188,8 @@ necessary to get to the terminus. characters in it, they will become part of the resulting keys - >>> dpath.util.new(x, 'a/b/e/f/g', "Roffle") - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> dpath.new(x, 'a/b/e/f/g', "Roffle") + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -211,9 +212,9 @@ object with None entries in order to make it big enough: .. code-block:: pycon - >>> dpath.util.new(x, 'a/b/e/f/h', []) - >>> dpath.util.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> dpath.new(x, 'a/b/e/f/h', []) + >>> dpath.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') + >>> print(json.dumps(x, indent=4, sort_keys=True)) { "a": { "b": { @@ -251,11 +252,11 @@ Handy! Example: Deleting Existing Keys =============================== -To delete keys in an object, use dpath.util.delete, which accepts the same globbing syntax as the other methods. +To delete keys in an object, use dpath.delete, which accepts the same globbing syntax as the other methods. .. code-block:: pycon - >>> help(dpath.util.delete) + >>> help(dpath.delete) delete(obj, glob, separator='/', afilter=None): Given a path glob, delete all elements that match the glob. @@ -266,84 +267,83 @@ To delete keys in an object, use dpath.util.delete, which accepts the same globb Example: Merging ================ -Also, check out dpath.util.merge. The python dict update() method is +Also, check out dpath.merge. The python dict update() method is great and all but doesn't handle merging dictionaries deeply. This one does. .. code-block:: pycon - >>> help(dpath.util.merge) - Help on function merge in module dpath.util: + >>> help(dpath.merge) + Help on function merge in module dpath: merge(dst, src, afilter=None, flags=4, _path='') Merge source into destination. Like dict.update() but performs deep merging. - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long + flags is an OR'ed combination of MergeType enum members. + * ADDITIVE : List objects are combined onto one long list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when + * REPLACE : Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different + * TYPESAFE : When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source replaces the destination in this situation. >>> y = {'a': {'b': { 'e': {'f': {'h': [None, 0, 1, None, 13, 14]}}}, 'c': 'RoffleWaffles'}} - >>> print json.dumps(y, indent=4, sort_keys=True) + >>> print(json.dumps(y, indent=4, sort_keys=True)) { - "a": { - "b": { - "e": { - "f": { - "h": [ - null, - 0, - 1, - null, - 13, - 14 - ] - } + "a": { + "b": { + "e": { + "f": { + "h": [ + null, + 0, + 1, + null, + 13, + 14 + ] + } + } + }, + "c": "RoffleWaffles" } - }, - "c": "RoffleWaffles" } - } - >>> dpath.util.merge(x, y) - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> dpath.merge(x, y) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { - "a": { - "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle", - "h": [ - null, - 0, - 1, - null, - 13, - 14, - null, - null, - null, - null, - null, - null, - null, - "Wow this is a big array, it sure is lonely in here by myself" - ] - } + "a": { + "b": { + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle", + "h": [ + null, + 0, + 1, + null, + 13, + 14, + null, + null, + null, + null, + null, + null, + null, + "Wow this is a big array, it sure is lonely in here by myself" + ] + } + } + }, + "c": "RoffleWaffles" } - }, - "c": "RoffleWaffles" - } } Now that's handy. You shouldn't try to use this as a replacement for the @@ -370,41 +370,41 @@ them: .. code-block:: pycon - >>> print json.dumps(x, indent=4, sort_keys=True) + >>> print(json.dumps(x, indent=4, sort_keys=True)) { - "a": { - "b": { - "3": 2, - "43": 30, - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle" + "a": { + "b": { + "3": 2, + "43": 30, + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle" + } + } } } - } - } } >>> def afilter(x): ... if "ffle" in str(x): ... return True ... return False ... - >>> result = dpath.util.search(x, '**', afilter=afilter) - >>> print json.dumps(result, indent=4, sort_keys=True) + >>> result = dpath.search(x, '**', afilter=afilter) + >>> print(json.dumps(result, indent=4, sort_keys=True)) { - "a": { - "b": { - "c": "Waffles", - "d": "Waffles", - "e": { - "f": { - "g": "Roffle" + "a": { + "b": { + "c": "Waffles", + "d": "Waffles", + "e": { + "f": { + "g": "Roffle" + } + } } } - } - } } Obviously filtering functions can perform more advanced tests (regular @@ -430,18 +430,18 @@ Separator got you down? Use lists as paths The default behavior in dpath is to assume that the path given is a string, which must be tokenized by splitting at the separator to yield a distinct set of path components against which dictionary keys can be individually glob tested. However, this presents a problem when you want to use paths that have a separator in their name; the tokenizer cannot properly understand what you mean by '/a/b/c' if it is possible for '/' to exist as a valid character in a key name. -To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.util.* support the use of a list instead of a string as a path. So for example: +To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.* support the use of a list instead of a string as a path. So for example: .. code-block:: python >>> x = { 'a': {'b/c': 0}} - >>> dpath.util.get(['a', 'b/c']) + >>> dpath.get(['a', 'b/c']) 0 dpath.segments : The Low-Level Backend ====================================== -dpath.util is where you want to spend your time: this library has the friendly +dpath is where you want to spend your time: this library has the friendly functions that will understand simple string globs, afilter functions, etc. dpath.segments is the backend pathing library. It passes around tuples of path @@ -451,12 +451,16 @@ components instead of string globs. :target: https://pypi.python.org/pypi/dpath/ :alt: PyPI: Latest Version +.. |Python Version| image:: https://img.shields.io/pypi/pyversions/dpath?style=flat + :target: https://pypi.python.org/pypi/dpath/ + :alt: Supported Python Version + .. |Build Status| image:: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml/badge.svg - :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml + :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml .. |Gitter| image:: https://badges.gitter.im/dpath-python/chat.svg - :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge - :alt: Gitter + :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge + :alt: Gitter Contributors ============ diff --git a/dpath/__init__.py b/dpath/__init__.py index 87f4114..d71bb71 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -1,5 +1,357 @@ -from dpath.ddict import DDict - __all__ = [ - DDict, + "new", + "delete", + "set", + "get", + "values", + "search", + "merge", + "exceptions", + "options", + "segments", + "types", + "version", + "MergeType", + "PathSegment", + "Filter", + "Glob", + "Path", + "Hints", + "Creator", + "DDict", ] + +from collections.abc import MutableMapping, MutableSequence +from typing import Union, List, Any, Callable, Optional + +from dpath import segments, options +from dpath.exceptions import InvalidKeyName, PathNotFound +from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints +from dpath.ddict import DDict + +_DEFAULT_SENTINEL = object() + + +def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: + """ + Given a path and separator, return a tuple of segments. If path is + already a non-leaf thing, return it. + + Note that a string path with the separator at index[0] will have the + separator stripped off. If you pass a list path, the separator is + ignored, and is assumed to be part of each key glob. It will not be + stripped. + """ + if not segments.leaf(path): + split_segments = path + else: + split_segments = path.lstrip(separator).split(separator) + + return split_segments + + +def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: + """ + Set the element at the terminus of path to value, and create + it if it does not exist (as opposed to 'set' that can only + change existing keys). + + path will NOT be treated like a glob. If it has globbing + characters in it, they will become part of the resulting + keys + + creator allows you to pass in a creator method that is + responsible for creating missing keys at arbitrary levels of + the path (see the help for dpath.path.set) + """ + split_segments = _split_path(path, separator) + if creator: + return segments.set(obj, split_segments, value, creator=creator) + return segments.set(obj, split_segments, value) + + +def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None) -> int: + """ + Given a obj, delete all elements that match the glob. + + Returns the number of deleted objects. Raises PathNotFound if no paths are + found to delete. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, value) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(value) and afilter(value) + + if (matched and not afilter) or selected: + key = path_segments[-1] + parent = segments.get(obj, path_segments[:-1]) + + # Deletion behavior depends on parent type + if isinstance(parent, MutableMapping): + del parent[key] + + else: + # Handle sequence types + # TODO: Consider cases where type isn't a simple list (e.g. set) + + if len(parent) - 1 == key: + # Removing the last element of a sequence. It can be + # truly removed without affecting the ordering of + # remaining items. + # + # Note: In order to achieve proper behavior we are + # relying on the reverse iteration of + # non-dictionaries from segments.kvs(). + # Otherwise we'd be unable to delete all the tails + # of a list and end up with None values when we + # don't need them. + del parent[key] + + else: + # This key can't be removed completely because it + # would affect the order of items that remain in our + # result. + parent[key] = None + + counter[0] += 1 + + [deleted] = segments.foldm(obj, f, [0]) + if not deleted: + raise PathNotFound(f"Could not find {glob} to delete it") + + return deleted + + +def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter = None) -> int: + """ + Given a path glob, set all existing elements in the document + to the given value. Returns the number of elements changed. + """ + globlist = _split_path(glob, separator) + + def f(obj, pair, counter): + (path_segments, found) = pair + + # Skip segments if they no longer exist in obj. + if not segments.has(obj, path_segments): + return + + matched = segments.match(path_segments, globlist) + selected = afilter and segments.leaf(found) and afilter(found) + + if (matched and not afilter) or (matched and selected): + segments.set(obj, path_segments, value, creator=None) + counter[0] += 1 + + [changed] = segments.foldm(obj, f, [0]) + return changed + + +def get( + obj: MutableMapping, + glob: Glob, + separator="/", + default: Any = _DEFAULT_SENTINEL +) -> Union[MutableMapping, object, Callable]: + """ + Given an object which contains only one possible match for the given glob, + return the value for the leaf matching the given glob. + If the glob is not found and a default is provided, + the default is returned. + + If more than one leaf matches the glob, ValueError is raised. If the glob is + not found and a default is not provided, KeyError is raised. + """ + if glob == "/": + return obj + + globlist = _split_path(glob, separator) + + def f(_, pair, results): + (path_segments, found) = pair + + if segments.match(path_segments, globlist): + results.append(found) + if len(results) > 1: + return False + + results = segments.fold(obj, f, []) + + if len(results) == 0: + if default is not _DEFAULT_SENTINEL: + return default + + raise KeyError(glob) + elif len(results) > 1: + raise ValueError(f"dpath.get() globs must match only one leaf: {glob}") + + return results[0] + + +def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None, dirs=True): + """ + Given an object and a path glob, return an array of all values which match + the glob. The arguments to this function are identical to those of search(). + """ + yielded = True + + return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] + + +def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): + """ + Given a path glob, return a dictionary containing all keys + that matched the given glob. + + If 'yielded' is true, then a dictionary will not be returned. + Instead tuples will be yielded in the form of (path, value) for + every element in the document that matched the glob. + """ + + split_glob = _split_path(glob, separator) + + def keeper(path, found): + """ + Generalized test for use in both yielded and folded cases. + Returns True if we want this result. Otherwise returns False. + """ + if not dirs and not segments.leaf(found): + return False + + matched = segments.match(path, split_glob) + selected = afilter and afilter(found) + + return (matched and not afilter) or (matched and selected) + + if yielded: + def yielder(): + for path, found in segments.walk(obj): + if keeper(path, found): + yield separator.join(map(segments.int_str, path)), found + + return yielder() + else: + def f(obj, pair, result): + (path, found) = pair + + if keeper(path, found): + segments.set(result, path, found, hints=segments.types(obj, path)) + + return segments.fold(obj, f, {}) + + +def merge(dst: MutableMapping, src: MutableMapping, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): + """ + Merge source into destination. Like dict.update() but performs deep + merging. + + NOTE: This does not do a deep copy of the source object. Applying merge + will result in references to src being present in the dst tree. If you do + not want src to potentially be modified by other changes in dst (e.g. more + merge calls), then use a deep copy of src. + + NOTE that merge() does NOT copy objects - it REFERENCES. If you merge + take these two dictionaries: + + >>> a = {'a': [0] } + >>> b = {'a': [1] } + + ... and you merge them into an empty dictionary, like so: + + >>> d = {} + >>> dpath.merge(d, a) + >>> dpath.merge(d, b) + + ... you might be surprised to find that a['a'] now contains [0, 1]. + This is because merge() says (d['a'] = a['a']), and thus creates a reference. + This reference is then modified when b is merged, causing both d and + a to have ['a'][0, 1]. To avoid this, make your own deep copies of source + objects that you intend to merge. For further notes see + https://github.com/akesterson/dpath-python/issues/58 + + flags is an OR'ed combination of MergeType enum members. + """ + filtered_src = search(src, '**', afilter=afilter, separator='/') + + def are_both_mutable(o1, o2): + mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) + seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) + + if mapP or seqP: + return True + + return False + + def merger(dst, src, _segments=()): + for key, found in segments.make_walkable(src): + # Our current path in the source. + current_path = _segments + (key,) + + if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: + raise InvalidKeyName("Empty string keys not allowed without " + "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " + f"{current_path}") + + # Validate src and dst types match. + if flags & MergeType.TYPESAFE: + if segments.has(dst, current_path): + target = segments.get(dst, current_path) + tt = type(target) + ft = type(found) + if tt != ft: + path = separator.join(current_path) + raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") + + # Path not present in destination, create it. + if not segments.has(dst, current_path): + segments.set(dst, current_path, found) + continue + + # Retrieve the value in the destination. + target = segments.get(dst, current_path) + + # If the types don't match, replace it. + if type(found) != type(target) and not are_both_mutable(found, target): + segments.set(dst, current_path, found) + continue + + # If target is a leaf, the replace it. + if segments.leaf(target): + segments.set(dst, current_path, found) + continue + + # At this point we know: + # + # * The target exists. + # * The types match. + # * The target isn't a leaf. + # + # Pretend we have a sequence and account for the flags. + try: + if flags & MergeType.ADDITIVE: + target += found + continue + + if flags & MergeType.REPLACE: + try: + target[""] + except TypeError: + segments.set(dst, current_path, found) + continue + except Exception: + raise + except Exception: + # We have a dictionary like thing and we need to attempt to + # recursively merge it. + merger(dst, found, current_path) + + merger(dst, filtered_src) + + return dst diff --git a/dpath/ddict.py b/dpath/ddict.py index 5097838..c472620 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -1,9 +1,6 @@ -from typing import Callable, Any, List, Dict +from typing import Any, MutableMapping -from dpath import util -from dpath.util import MERGE_ADDITIVE - -FilterType = Callable[[Any], bool] # (Any) -> bool +from dpath import MergeType, merge, search, values, get, set, Filter, Glob _DEFAULT_SENTINEL: Any = object() @@ -16,28 +13,28 @@ class DDict(dict): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def set(self, glob: str, value, separator="/", afilter: FilterType = None): - return util.set(self, glob, value, separator=separator, afilter=afilter) + def set(self, glob: Glob, value, separator="/", afilter: Filter = None): + return set(self, glob, value, separator=separator, afilter=afilter) - def get(self, glob: str, separator="/", default=_DEFAULT_SENTINEL) -> Any: + def get(self, glob: Glob, separator="/", default=_DEFAULT_SENTINEL) -> Any: """ Same as dict.get but glob aware """ if default is not _DEFAULT_SENTINEL: # Default value was passed - return util.get(self, glob, separator=separator, default=default) + return get(self, glob, separator=separator, default=default) else: # Let util.get handle default value - return util.get(self, glob, separator=separator) + return get(self, glob, separator=separator) - def values(self, glob="*", separator="/", afilter: FilterType = None, dirs=True) -> List: + def values(self, glob: Glob = "*", separator="/", afilter: Filter = None, dirs=True): """ Same as dict.values but glob aware """ - return util.values(self, glob, separator=separator, afilter=afilter, dirs=dirs) + return values(self, glob, separator=separator, afilter=afilter, dirs=dirs) - def search(self, glob, yielded=False, separator="/", afilter: FilterType = None, dirs=True): - return util.search(self, glob, yielded=yielded, separator=separator, afilter=afilter, dirs=dirs) + def search(self, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): + return search(self, glob, yielded=yielded, separator=separator, afilter=afilter, dirs=dirs) - def merge(self, src: Dict, separator="/", afilter: FilterType = None, flags=MERGE_ADDITIVE): - return util.merge(self, src, separator=separator, afilter=afilter, flags=flags) + def merge(self, src: MutableMapping, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): + return merge(self, src, separator=separator, afilter=afilter, flags=flags) diff --git a/dpath/segments.py b/dpath/segments.py index 1016a06..faa763f 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,15 +1,22 @@ from copy import deepcopy -from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound -from dpath import options from fnmatch import fnmatchcase +from typing import List, Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping + +from dpath import options +from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound +from dpath.types import PathSegment, Creator, Hints -def kvs(node): - ''' - Return a (key, value) iterator for the node. +def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: + """ + Returns an iterator which yields tuple pairs of (node index, node value), regardless of node type. - kvs(node) -> (generator -> (key, value)) - ''' + * For dict nodes `node.items()` will be returned. + * For sequence nodes (lists/tuples/etc.) a zip between index number and index value will be returned. + * Edge cases will result in an empty iterator being returned. + + make_walkable(node) -> (generator -> (key, value)) + """ try: return iter(node.items()) except AttributeError: @@ -23,23 +30,19 @@ def kvs(node): def leaf(thing): - ''' + """ Return True if thing is a leaf, otherwise False. - - leaf(thing) -> bool - ''' + """ leaves = (bytes, str, int, float, bool, type(None)) return isinstance(thing, leaves) def leafy(thing): - ''' + """ Same as leaf(thing), but also treats empty sequences and dictionaries as True. - - leafy(thing) -> bool - ''' + """ try: return leaf(thing) or len(thing) == 0 @@ -49,52 +52,53 @@ def leafy(thing): def walk(obj, location=()): - ''' + """ Yield all valid (segments, value) pairs (from a breadth-first search, right-to-left on sequences). walk(obj) -> (generator -> (segments, value)) - ''' + """ if not leaf(obj): - for k, v in kvs(obj): + for k, v in make_walkable(obj): length = None try: length = len(k) - except: + except TypeError: pass if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: raise InvalidKeyName("Empty string keys not allowed without " "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(location + (k,))) - yield ((location + (k,)), v) - for k, v in kvs(obj): + f"{location + (k,)}") + yield (location + (k,)), v + + for k, v in make_walkable(obj): for found in walk(v, location + (k,)): yield found def get(obj, segments): - ''' + """ Return the value at the path indicated by segments. get(obj, segments) -> value - ''' + """ current = obj - for (i, segment) in enumerate(segments): + for i, segment in enumerate(segments): if leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") current = current[segment] return current def has(obj, segments): - ''' + """ Return True if the path exists in the obj. Otherwise return False. has(obj, segments) -> bool - ''' + """ try: get(obj, segments) return True @@ -103,24 +107,24 @@ def has(obj, segments): def expand(segments): - ''' + """ Yield a tuple of segments for each possible length of segments. Starting from the shortest length of segments and increasing by 1. expand(keys) -> (..., keys[:-2], keys[:-1]) - ''' + """ index = 0 - for segment in segments: + for _ in segments: index += 1 yield segments[:index] def types(obj, segments): - ''' + """ For each segment produce a tuple of (segment, type(value)). types(obj, segments) -> ((segment[0], type0), (segment[1], type1), ...) - ''' + """ result = [] for depth in expand(segments): result.append((depth[-1], type(get(obj, depth)))) @@ -128,39 +132,39 @@ def types(obj, segments): def leaves(obj): - ''' + """ Yield all leaves as (segment, value) pairs. leaves(obj) -> (generator -> (segment, value)) - ''' + """ return filter(lambda p: leafy(p[1]), walk(obj)) -def int_str(segment): - ''' +def int_str(segment: PathSegment) -> PathSegment: + """ If the segment is an integer, return the string conversion. Otherwise return the segment unchanged. The conversion uses 'str'. int_str(segment) -> str - ''' + """ if isinstance(segment, int): return str(segment) return segment class Star(object): - ''' + """ Used to create a global STAR symbol for tracking stars added when expanding star-star globs. - ''' + """ pass STAR = Star() -def match(segments, glob): - ''' +def match(segments: Sequence[PathSegment], glob: Sequence[str]): + """ Return True if the segments match the given glob, otherwise False. For the purposes of matching, integers are converted to their string @@ -177,23 +181,22 @@ def match(segments, glob): throws an exception the result will be False. match(segments, glob) -> bool - ''' + """ segments = tuple(segments) glob = tuple(glob) path_len = len(segments) glob_len = len(glob) - # Index of the star-star in the glob. - ss = -1 # The star-star normalized glob ('**' has been removed). ss_glob = glob if '**' in glob: + # Index of the star-star in the glob. ss = glob.index('**') + if '**' in glob[ss + 1:]: - raise InvalidGlob("Invalid glob. Only one '**' is permitted per glob: {}" - "".format(glob)) + raise InvalidGlob(f"Invalid glob. Only one '**' is permitted per glob: {glob}") # Convert '**' segment into multiple '*' segments such that the # lengths of the path and glob match. '**' also can collapse and @@ -211,14 +214,14 @@ def match(segments, glob): # If we were successful in matching up the lengths, then we can # compare them using fnmatch. if path_len == len(ss_glob): - for (s, g) in zip(map(int_str, segments), map(int_str, ss_glob)): + for s, g in zip(map(int_str, segments), map(int_str, ss_glob)): # Match the stars we added to the glob to the type of the # segment itself. if g is STAR: if isinstance(s, bytes): g = b'*' else: - g = u'*' + g = '*' # Let's see if the glob matches. We will turn any kind of # exception while attempting to match into a False for the @@ -237,15 +240,15 @@ def match(segments, glob): return False -def extend(thing, index, value=None): - ''' +def extend(thing: List, index: int, value=None): + """ Extend a sequence like thing such that it contains at least index + 1 many elements. The extension values will be None (default). extend(thing, int) -> [thing..., None, ...] - ''' + """ try: - expansion = (type(thing)()) + expansion = type(thing)() # Using this rather than the multiply notation in order to support a # wider variety of sequence like things. @@ -262,13 +265,18 @@ def extend(thing, index, value=None): return thing -def __default_creator__(current, segments, i, hints=()): - ''' +def _default_creator( + current: Union[MutableMapping, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () +): + """ Create missing path components. If the segment is an int, then it will create a list. Otherwise a dictionary is created. set(obj, segments, value) -> obj - ''' + """ segment = segments[i] length = len(segments) @@ -292,20 +300,31 @@ def __default_creator__(current, segments, i, hints=()): current[segment] = {} -def set(obj, segments, value, creator=__default_creator__, hints=()): - ''' +def set( + obj: MutableMapping, + segments: Sequence[PathSegment], + value, + creator: Optional[Creator] = _default_creator, + hints: Hints = () +) -> MutableMapping: + """ Set the value in obj at the place indicated by segments. If creator is not - None (default __default_creator__), then call the creator function to + None (default _default_creator), then call the creator function to create any missing path components. set(obj, segments, value) -> obj - ''' + """ current = obj length = len(segments) # For everything except the last value, walk down the path and # create if creator is set. for (i, segment) in enumerate(segments[:-1]): + + # If segment is non-int but supposed to be a sequence index + if isinstance(segment, str) and isinstance(current, Sequence) and segment.isdigit(): + segment = int(segment) + try: # Optimistically try to get the next value. This makes the # code agnostic to whether current is a list or a dict. @@ -314,24 +333,30 @@ def set(obj, segments, value, creator=__default_creator__, hints=()): current[segment] except: if creator is not None: - creator(current, segments, i, hints=hints) + creator(current, segments, i, hints) else: raise current = current[segment] if i != length - 1 and leaf(current): - raise PathNotFound('Path: {}[{}]'.format(segments, i)) + raise PathNotFound(f"Path: {segments}[{i}]") + + last_segment = segments[-1] - if isinstance(segments[-1], int): - extend(current, segments[-1]) + # Resolve ambiguity of last segment + if isinstance(last_segment, str) and isinstance(current, Sequence) and last_segment.isdigit(): + last_segment = int(last_segment) - current[segments[-1]] = value + if isinstance(last_segment, int): + extend(current, last_segment) + + current[last_segment] = value return obj def fold(obj, f, acc): - ''' + """ Walk obj applying f to each path and returning accumulator acc. The function f will be called, for each result in walk(obj): @@ -343,7 +368,7 @@ def fold(obj, f, acc): retrieved from the walk. fold(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ for pair in walk(obj): if f(obj, pair, acc) is False: break @@ -351,33 +376,34 @@ def fold(obj, f, acc): def foldm(obj, f, acc): - ''' + """ Same as fold(), but permits mutating obj. This requires all paths in walk(obj) to be loaded into memory (whereas fold does not). foldm(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc - ''' + """ pairs = tuple(walk(obj)) for pair in pairs: - (segments, value) = pair if f(obj, pair, acc) is False: break return acc def view(obj, glob): - ''' + """ Return a view of the object where the glob matches. A view retains the same form as the obj, but is limited to only the paths that matched. Views are new objects (a deepcopy of the matching values). view(obj, glob) -> obj' - ''' + """ + def f(obj, pair, result): (segments, value) = pair if match(segments, glob): if not has(result, segments): set(result, segments, deepcopy(value), hints=types(obj, segments)) + return fold(obj, f, type(obj)()) diff --git a/dpath/types.py b/dpath/types.py new file mode 100644 index 0000000..b876e6a --- /dev/null +++ b/dpath/types.py @@ -0,0 +1,45 @@ +from enum import IntFlag, auto +from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping + + +class MergeType(IntFlag): + ADDITIVE = auto() + """List objects are combined onto one long list (NOT a set). This is the default flag.""" + + REPLACE = auto() + """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination \ + with the source.""" + + TYPESAFE = auto() + """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source \ + replaces the destination in this situation.""" + + +PathSegment = Union[int, str] +"""Type alias for dict path segments where integers are explicitly casted.""" + +Filter = Callable[[Any], bool] +"""Type alias for filter functions. + +(Any) -> bool""" + +Glob = Union[str, Sequence[str]] +"""Type alias for glob parameters.""" + +Path = Union[str, Sequence[PathSegment]] +"""Type alias for path parameters.""" + +Hints = Sequence[Tuple[PathSegment, type]] +"""Type alias for creator function hint sequences.""" + +Creator = Callable[[Union[MutableMapping, List], Path, int, Optional[Hints]], None] +"""Type alias for creator functions. + +Example creator function signature: + + def creator( + current: Union[MutableMapping, List], + segments: Sequence[PathSegment], + i: int, + hints: Sequence[Tuple[PathSegment, type]] = () + )""" diff --git a/dpath/util.py b/dpath/util.py index 48c03be..60d0319 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,354 +1,51 @@ -from collections.abc import MutableMapping -from collections.abc import MutableSequence -from dpath import options -from dpath.exceptions import InvalidKeyName -import dpath.segments +import warnings -_DEFAULT_SENTINAL = object() -MERGE_REPLACE = (1 << 1) -MERGE_ADDITIVE = (1 << 2) -MERGE_TYPESAFE = (1 << 3) +import dpath +from dpath import _DEFAULT_SENTINEL +from dpath.types import MergeType -def __safe_path__(path, separator): - ''' - Given a path and separator, return a tuple of segments. If path is - already a non-leaf thing, return it. +def deprecated(func): + message = \ + "The dpath.util package is being deprecated. All util functions have been moved to dpath package top level." - Note that a string path with the separator at index[0] will have the - separator stripped off. If you pass a list path, the separator is - ignored, and is assumed to be part of each key glob. It will not be - stripped. - ''' - if not dpath.segments.leaf(path): - segments = path - else: - segments = path.lstrip(separator).split(separator) + def wrapper(*args, **kwargs): + warnings.warn(message, DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) - # FIXME: This check was in the old internal library, but I can't - # see a way it could fail... - for i, segment in enumerate(segments): - if (separator and (separator in segment)): - raise InvalidKeyName("{} at {}[{}] contains the separator '{}'" - "".format(segment, segments, i, separator)) + return wrapper - # Attempt to convert integer segments into actual integers. - final = [] - for segment in segments: - try: - final.append(int(segment)) - except: - final.append(segment) - segments = final - return segments +@deprecated +def new(obj, path, value, separator="/", creator=None): + return dpath.new(obj, path, value, separator, creator) -def new(obj, path, value, separator='/', creator=None): - ''' - Set the element at the terminus of path to value, and create - it if it does not exist (as opposed to 'set' that can only - change existing keys). +@deprecated +def delete(obj, glob, separator="/", afilter=None): + return dpath.delete(obj, glob, separator, afilter) - path will NOT be treated like a glob. If it has globbing - characters in it, they will become part of the resulting - keys - creator allows you to pass in a creator method that is - responsible for creating missing keys at arbitrary levels of - the path (see the help for dpath.path.set) - ''' - segments = __safe_path__(path, separator) - if creator: - return dpath.segments.set(obj, segments, value, creator=creator) - return dpath.segments.set(obj, segments, value) +@deprecated +def set(obj, glob, value, separator="/", afilter=None): + return dpath.set(obj, glob, value, separator, afilter) -def delete(obj, glob, separator='/', afilter=None): - ''' - Given a obj, delete all elements that match the glob. +@deprecated +def get(obj, glob, separator="/", default=_DEFAULT_SENTINEL): + return dpath.get(obj, glob, separator, default) - Returns the number of deleted objects. Raises PathNotFound if no paths are - found to delete. - ''' - globlist = __safe_path__(glob, separator) - def f(obj, pair, counter): - (segments, value) = pair +@deprecated +def values(obj, glob, separator="/", afilter=None, dirs=True): + return dpath.values(obj, glob, separator, afilter, dirs) - # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): - return - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(value) and afilter(value) +@deprecated +def search(obj, glob, yielded=False, separator="/", afilter=None, dirs=True): + return dpath.search(obj, glob, yielded, separator, afilter, dirs) - if (matched and not afilter) or selected: - key = segments[-1] - parent = dpath.segments.get(obj, segments[:-1]) - try: - # Attempt to treat parent like a sequence. - parent[0] - - if len(parent) - 1 == key: - # Removing the last element of a sequence. It can be - # truly removed without affecting the ordering of - # remaining items. - # - # Note: In order to achieve proper behavior we are - # relying on the reverse iteration of - # non-dictionaries from dpath.segments.kvs(). - # Otherwise we'd be unable to delete all the tails - # of a list and end up with None values when we - # don't need them. - del parent[key] - else: - # This key can't be removed completely because it - # would affect the order of items that remain in our - # result. - parent[key] = None - except: - # Attempt to treat parent like a dictionary instead. - del parent[key] - - counter[0] += 1 - - [deleted] = dpath.segments.foldm(obj, f, [0]) - if not deleted: - raise dpath.exceptions.PathNotFound("Could not find {0} to delete it".format(glob)) - - return deleted - - -def set(obj, glob, value, separator='/', afilter=None): - ''' - Given a path glob, set all existing elements in the document - to the given value. Returns the number of elements changed. - ''' - globlist = __safe_path__(glob, separator) - - def f(obj, pair, counter): - (segments, found) = pair - - # Skip segments if they no longer exist in obj. - if not dpath.segments.has(obj, segments): - return - - matched = dpath.segments.match(segments, globlist) - selected = afilter and dpath.segments.leaf(found) and afilter(found) - - if (matched and not afilter) or (matched and selected): - dpath.segments.set(obj, segments, value, creator=None) - counter[0] += 1 - - [changed] = dpath.segments.foldm(obj, f, [0]) - return changed - - -def get(obj, glob, separator='/', default=_DEFAULT_SENTINAL): - ''' - Given an object which contains only one possible match for the given glob, - return the value for the leaf matching the given glob. - If the glob is not found and a default is provided, - the default is returned. - - If more than one leaf matches the glob, ValueError is raised. If the glob is - not found and a default is not provided, KeyError is raised. - ''' - if glob == '/': - return obj - - globlist = __safe_path__(glob, separator) - - def f(obj, pair, results): - (segments, found) = pair - - if dpath.segments.match(segments, globlist): - results.append(found) - if len(results) > 1: - return False - - results = dpath.segments.fold(obj, f, []) - - if len(results) == 0: - if default is not _DEFAULT_SENTINAL: - return default - - raise KeyError(glob) - elif len(results) > 1: - raise ValueError("dpath.util.get() globs must match only one leaf : %s" % glob) - - return results[0] - - -def values(obj, glob, separator='/', afilter=None, dirs=True): - ''' - Given an object and a path glob, return an array of all values which match - the glob. The arguments to this function are identical to those of search(). - ''' - yielded = True - - return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] - - -def search(obj, glob, yielded=False, separator='/', afilter=None, dirs=True): - ''' - Given a path glob, return a dictionary containing all keys - that matched the given glob. - - If 'yielded' is true, then a dictionary will not be returned. - Instead tuples will be yielded in the form of (path, value) for - every element in the document that matched the glob. - ''' - - globlist = __safe_path__(glob, separator) - - def keeper(segments, found): - ''' - Generalized test for use in both yielded and folded cases. - Returns True if we want this result. Otherwise returns False. - ''' - if not dirs and not dpath.segments.leaf(found): - return False - - matched = dpath.segments.match(segments, globlist) - selected = afilter and afilter(found) - - return (matched and not afilter) or (matched and selected) - - if yielded: - def yielder(): - for segments, found in dpath.segments.walk(obj): - if keeper(segments, found): - yield (separator.join(map(dpath.segments.int_str, segments)), found) - return yielder() - else: - def f(obj, pair, result): - (segments, found) = pair - - if keeper(segments, found): - dpath.segments.set(result, segments, found, hints=dpath.segments.types(obj, segments)) - - return dpath.segments.fold(obj, f, {}) - - -def merge(dst, src, separator='/', afilter=None, flags=MERGE_ADDITIVE): - ''' - Merge source into destination. Like dict.update() but performs deep - merging. - - NOTE: This does not do a deep copy of the source object. Applying merge - will result in references to src being present in the dst tree. If you do - not want src to potentially be modified by other changes in dst (e.g. more - merge calls), then use a deep copy of src. - - NOTE that merge() does NOT copy objects - it REFERENCES. If you merge - take these two dictionaries: - - >>> a = {'a': [0] } - >>> b = {'a': [1] } - - ... and you merge them into an empty dictionary, like so: - - >>> d = {} - >>> dpath.util.merge(d, a) - >>> dpath.util.merge(d, b) - - ... you might be surprised to find that a['a'] now contains [0, 1]. - This is because merge() says (d['a'] = a['a']), and thus creates a reference. - This reference is then modified when b is merged, causing both d and - a to have ['a'][0, 1]. To avoid this, make your own deep copies of source - objects that you intend to merge. For further notes see - https://github.com/akesterson/dpath-python/issues/58 - - flags is an OR'ed combination of MERGE_ADDITIVE, MERGE_REPLACE, - MERGE_TYPESAFE. - * MERGE_ADDITIVE : List objects are combined onto one long - list (NOT a set). This is the default flag. - * MERGE_REPLACE : Instead of combining list objects, when - 2 list objects are at an equal depth of merge, replace - the destination with the source. - * MERGE_TYPESAFE : When 2 keys at equal levels are of different - types, raise a TypeError exception. By default, the source - replaces the destination in this situation. - ''' - filtered_src = search(src, '**', afilter=afilter, separator='/') - - def are_both_mutable(o1, o2): - mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) - seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) - - if mapP or seqP: - return True - - return False - - def merger(dst, src, _segments=()): - for key, found in dpath.segments.kvs(src): - # Our current path in the source. - segments = _segments + (key,) - - if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: - raise InvalidKeyName("Empty string keys not allowed without " - "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " - "{}".format(segments)) - - # Validate src and dst types match. - if flags & MERGE_TYPESAFE: - if dpath.segments.has(dst, segments): - target = dpath.segments.get(dst, segments) - tt = type(target) - ft = type(found) - if tt != ft: - path = separator.join(segments) - raise TypeError("Cannot merge objects of type" - "{0} and {1} at {2}" - "".format(tt, ft, path)) - - # Path not present in destination, create it. - if not dpath.segments.has(dst, segments): - dpath.segments.set(dst, segments, found) - continue - - # Retrieve the value in the destination. - target = dpath.segments.get(dst, segments) - - # If the types don't match, replace it. - if ((type(found) != type(target)) and (not are_both_mutable(found, target))): - dpath.segments.set(dst, segments, found) - continue - - # If target is a leaf, the replace it. - if dpath.segments.leaf(target): - dpath.segments.set(dst, segments, found) - continue - - # At this point we know: - # - # * The target exists. - # * The types match. - # * The target isn't a leaf. - # - # Pretend we have a sequence and account for the flags. - try: - if flags & MERGE_ADDITIVE: - target += found - continue - - if flags & MERGE_REPLACE: - try: - target[''] - except TypeError: - dpath.segments.set(dst, segments, found) - continue - except: - raise - except: - # We have a dictionary like thing and we need to attempt to - # recursively merge it. - merger(dst, found, segments) - - merger(dst, filtered_src) - - return dst +@deprecated +def merge(dst, src, separator="/", afilter=None, flags=MergeType.ADDITIVE): + return dpath.merge(dst, src, separator, afilter, flags), diff --git a/dpath/version.py b/dpath/version.py index 215732a..5b0431e 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.0.4" +VERSION = "2.1.1" diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000..92830d3 --- /dev/null +++ b/flake8.ini @@ -0,0 +1,5 @@ +[flake8] +filename= + setup.py, + dpath/, + tests/ diff --git a/setup.py b/setup.py index abce1a3..a6891d6 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ -from distutils.core import setup -import dpath.version import os +from distutils.core import setup +import dpath.version long_description = open( os.path.join( @@ -13,7 +13,7 @@ if __name__ == "__main__": setup( name="dpath", - url="https://www.github.com/akesterson/dpath-python", + url="https://github.com/dpath-maintainers/dpath-python", version=dpath.version.VERSION, description="Filesystem-like pathing and searching for dictionaries", long_description=long_description, @@ -25,7 +25,17 @@ scripts=[], packages=["dpath"], data_files=[], - python_requires=">=3", + + # Type hints are great. + # Function annotations were added in Python 3.0. + # Typing module was added in Python 3.5. + # Variable annotations were added in Python 3.6. + # Python versions that are >=3.6 are more popular. + # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) + # + # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. + # 3.6 was dropped because of EOL and this issue: https://github.com/actions/setup-python/issues/544 + python_requires=">=3.7", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e188ff3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import warnings + +warnings.simplefilter("always", DeprecationWarning) diff --git a/tests/test_broken_afilter.py b/tests/test_broken_afilter.py index a59454e..683c727 100644 --- a/tests/test_broken_afilter.py +++ b/tests/test_broken_afilter.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath import sys @@ -25,15 +25,15 @@ def afilter(x): 'a/b/c/f', ] - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c']) + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c']) + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] def filter(x): sys.stderr.write(str(x)) @@ -52,7 +52,7 @@ def filter(x): ], } - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', yielded=True)] - results = [[x[0], x[1]] for x in dpath.util.search(a, 'actions/*', afilter=filter, yielded=True)] - assert(len(results) == 1) - assert(results[0][1]['type'] == 'correct') + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', yielded=True)] + results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', afilter=filter, yielded=True)] + assert len(results) == 1 + assert results[0][1]['type'] == 'correct' diff --git a/tests/test_util_delete.py b/tests/test_delete.py similarity index 54% rename from tests/test_util_delete.py rename to tests/test_delete.py index c14acf7..c7879b0 100644 --- a/tests/test_util_delete.py +++ b/tests/test_delete.py @@ -1,5 +1,6 @@ -from nose.tools import raises -import dpath.util +from nose2.tools.such import helper + +import dpath import dpath.exceptions @@ -10,8 +11,8 @@ def test_delete_separator(): }, } - dpath.util.delete(dict, ';a;b', separator=";") - assert('b' not in dict['a']) + dpath.delete(dict, ';a;b', separator=";") + assert 'b' not in dict['a'] def test_delete_existing(): @@ -21,18 +22,18 @@ def test_delete_existing(): }, } - dpath.util.delete(dict, '/a/b') - assert('b' not in dict['a']) + dpath.delete(dict, '/a/b') + assert 'b' not in dict['a'] -@raises(dpath.exceptions.PathNotFound) def test_delete_missing(): dict = { "a": { }, } - dpath.util.delete(dict, '/a/b') + with helper.assertRaises(dpath.exceptions.PathNotFound): + dpath.delete(dict, '/a/b') def test_delete_filter(): @@ -49,7 +50,7 @@ def afilter(x): }, } - dpath.util.delete(dict, '/a/*', afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert ('d' not in dict['a']) + dpath.delete(dict, '/a/*', afilter=afilter) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert 'd' not in dict['a'] diff --git a/tests/test_util_get_values.py b/tests/test_get_values.py similarity index 54% rename from tests/test_util_get_values.py rename to tests/test_get_values.py index 5a14f1e..9eeef82 100644 --- a/tests/test_util_get_values.py +++ b/tests/test_get_values.py @@ -1,20 +1,21 @@ -from nose.tools import assert_raises - import datetime import decimal -import dpath.util -import mock import time +import mock +from nose2.tools.such import helper + +import dpath + def test_util_get_root(): x = {'p': {'a': {'t': {'h': 'value'}}}} - ret = dpath.util.get(x, '/p/a/t/h') - assert(ret == 'value') + ret = dpath.get(x, '/p/a/t/h') + assert ret == 'value' - ret = dpath.util.get(x, '/') - assert(ret == x) + ret = dpath.get(x, '/') + assert ret == x def test_get_explicit_single(): @@ -30,11 +31,11 @@ def test_get_explicit_single(): }, } - assert(dpath.util.get(ehash, '/a/b/c/f') == 2) - assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f']) == 2) - assert(dpath.util.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2) - assert(dpath.util.get(ehash, ['does', 'not', 'exist'], default=None) is None) - assert(dpath.util.get(ehash, ['doesnt', 'exist'], default=5) == 5) + assert dpath.get(ehash, '/a/b/c/f') == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 + assert dpath.get(ehash, ['does', 'not', 'exist'], default=None) is None + assert dpath.get(ehash, ['doesnt', 'exist'], default=5) == 5 def test_get_glob_single(): @@ -50,10 +51,10 @@ def test_get_glob_single(): }, } - assert(dpath.util.get(ehash, '/a/b/*/f') == 2) - assert(dpath.util.get(ehash, ['a', 'b', '*', 'f']) == 2) - assert(dpath.util.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2) - assert(dpath.util.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6) + assert dpath.get(ehash, '/a/b/*/f') == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f']) == 2 + assert dpath.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 + assert dpath.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 def test_get_glob_multiple(): @@ -70,16 +71,16 @@ def test_get_glob_multiple(): }, } - assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd'], default=3) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd'], default=3) def test_get_absent(): ehash = {} - assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_values(): @@ -95,38 +96,38 @@ def test_values(): }, } - ret = dpath.util.values(ehash, '/a/b/c/*') - assert(isinstance(ret, list)) - assert(0 in ret) - assert(1 in ret) - assert(2 in ret) + ret = dpath.values(ehash, '/a/b/c/*') + assert isinstance(ret, list) + assert 0 in ret + assert 1 in ret + assert 2 in ret - ret = dpath.util.values(ehash, ['a', 'b', 'c', '*']) - assert(isinstance(ret, list)) - assert(0 in ret) - assert(1 in ret) - assert(2 in ret) + ret = dpath.values(ehash, ['a', 'b', 'c', '*']) + assert isinstance(ret, list) + assert 0 in ret + assert 1 in ret + assert 2 in ret -@mock.patch('dpath.util.search') +@mock.patch('dpath.search') def test_values_passes_through(searchfunc): searchfunc.return_value = [] def y(): - pass + return False - dpath.util.values({}, '/a/b', ':', y, False) + dpath.values({}, '/a/b', ':', y, False) searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) - dpath.util.values({}, ['a', 'b'], ':', y, False) + dpath.values({}, ['a', 'b'], ':', y, False) searchfunc.assert_called_with({}, ['a', 'b'], True, ':', y, False) def test_none_values(): d = {'p': {'a': {'t': {'h': None}}}} - v = dpath.util.get(d, 'p/a/t/h') - assert(v is None) + v = dpath.get(d, 'p/a/t/h') + assert v is None def test_values_list(): @@ -141,9 +142,9 @@ def test_values_list(): ], } - ret = dpath.util.values(a, 'actions/*') - assert(isinstance(ret, list)) - assert(len(ret) == 2) + ret = dpath.values(a, 'actions/*') + assert isinstance(ret, list) + assert len(ret) == 2 def test_non_leaf_leaf(): @@ -173,18 +174,18 @@ def func(x): } # It should be possible to get the callables: - assert dpath.util.get(testdict, 'a') == func - assert dpath.util.get(testdict, 'b')(42) == 42 + assert dpath.get(testdict, 'a') == func + assert dpath.get(testdict, 'b')(42) == 42 # It should be possible to get other values: - assert dpath.util.get(testdict, 'c/0') == testdict['c'][0] - assert dpath.util.get(testdict, 'd')[0] == testdict['d'][0] - assert dpath.util.get(testdict, 'd/0') == testdict['d'][0] - assert dpath.util.get(testdict, 'd/1') == testdict['d'][1] - assert dpath.util.get(testdict, 'e') == testdict['e'] + assert dpath.get(testdict, 'c/0') == testdict['c'][0] + assert dpath.get(testdict, 'd')[0] == testdict['d'][0] + assert dpath.get(testdict, 'd/0') == testdict['d'][0] + assert dpath.get(testdict, 'd/1') == testdict['d'][1] + assert dpath.get(testdict, 'e') == testdict['e'] # Values should also still work: - assert dpath.util.values(testdict, 'f/config') == ['something'] + assert dpath.values(testdict, 'f/config') == ['something'] # Data classes should also be retrievable: try: @@ -206,4 +207,4 @@ class Connection: ), } - assert dpath.util.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] + assert dpath.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] diff --git a/tests/test_util_merge.py b/tests/test_merge.py similarity index 59% rename from tests/test_util_merge.py rename to tests/test_merge.py index 968a4fa..a8b638c 100644 --- a/tests/test_util_merge.py +++ b/tests/test_merge.py @@ -1,9 +1,10 @@ -import nose import copy -from nose.tools import raises +from nose2.tools.such import helper -import dpath.util + +import dpath +from dpath import MergeType def test_merge_typesafe_and_separator(): @@ -19,9 +20,9 @@ def test_merge_typesafe_and_separator(): } try: - dpath.util.merge(dst, src, flags=(dpath.util.MERGE_ADDITIVE | dpath.util.MERGE_TYPESAFE), separator=";") + dpath.merge(dst, src, flags=(dpath.MergeType.ADDITIVE | dpath.MergeType.TYPESAFE), separator=";") except TypeError as e: - assert(str(e).endswith("dict;integer")) + assert str(e).endswith("dict;integer") return raise Exception("MERGE_TYPESAFE failed to raise an exception when merging between str and int!") @@ -35,8 +36,8 @@ def test_merge_simple_int(): "integer": 3, } - dpath.util.merge(dst, src) - nose.tools.eq_(dst["integer"], src["integer"]) + dpath.merge(dst, src) + assert dst["integer"] == src["integer"], "%r != %r" % (dst["integer"], src["integer"]) def test_merge_simple_string(): @@ -47,8 +48,8 @@ def test_merge_simple_string(): "string": "lol I am a string", } - dpath.util.merge(dst, src) - nose.tools.eq_(dst["string"], src["string"]) + dpath.merge(dst, src) + assert dst["string"] == src["string"], "%r != %r" % (dst["string"], src["string"]) def test_merge_simple_list_additive(): @@ -59,8 +60,8 @@ def test_merge_simple_list_additive(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_ADDITIVE) - nose.tools.eq_(dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) + dpath.merge(dst, src, flags=MergeType.ADDITIVE) + assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) def test_merge_simple_list_replace(): @@ -71,8 +72,8 @@ def test_merge_simple_list_replace(): "list": [0, 1, 2, 3], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) - nose.tools.eq_(dst["list"], [7, 8, 9, 10]) + dpath.merge(dst, src, flags=dpath.MergeType.REPLACE) + assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) def test_merge_simple_dict(): @@ -87,8 +88,8 @@ def test_merge_simple_dict(): }, } - dpath.util.merge(dst, src) - nose.tools.eq_(dst["dict"]["key"], src["dict"]["key"]) + dpath.merge(dst, src) + assert dst["dict"]["key"] == src["dict"]["key"], "%r != %r" % (dst["dict"]["key"], src["dict"]["key"]) def test_merge_filter(): @@ -106,13 +107,12 @@ def afilter(x): } dst = {} - dpath.util.merge(dst, src, afilter=afilter) - assert ("key2" in dst) - assert ("key" not in dst) - assert ("otherdict" not in dst) + dpath.merge(dst, src, afilter=afilter) + assert "key2" in dst + assert "key" not in dst + assert "otherdict" not in dst -@raises(TypeError) def test_merge_typesafe(): src = { "dict": { @@ -123,10 +123,9 @@ def test_merge_typesafe(): ], } - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) -@raises(TypeError) def test_merge_mutables(): class tcid(dict): pass @@ -150,28 +149,28 @@ class tcis(list): "ms": tcis(['a', 'b', 'c']), } - dpath.util.merge(dst, src) + dpath.merge(dst, src) print(dst) - assert(dst["mm"]["a"] == src["mm"]["a"]) - assert(dst['ms'][2] == 'c') - assert("casserole" in dst["mm"]) + assert dst["mm"]["a"] == src["mm"]["a"] + assert dst['ms'][2] == 'c' + assert "casserole" in dst["mm"] - dpath.util.merge(dst, src, flags=dpath.util.MERGE_TYPESAFE) + helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) def test_merge_replace_1(): dct_a = {"a": {"b": [1, 2, 3]}} dct_b = {"a": {"b": [1]}} - dpath.util.merge(dct_a, dct_b, flags=dpath.util.MERGE_REPLACE) - assert(len(dct_a['a']['b']) == 1) + dpath.merge(dct_a, dct_b, flags=dpath.MergeType.REPLACE) + assert len(dct_a['a']['b']) == 1 def test_merge_replace_2(): d1 = {'a': [0, 1, 2]} d2 = {'a': ['a']} - dpath.util.merge(d1, d2, flags=dpath.util.MERGE_REPLACE) - assert(len(d1['a']) == 1) - assert(d1['a'][0] == 'a') + dpath.merge(d1, d2, flags=dpath.MergeType.REPLACE) + assert len(d1['a']) == 1 + assert d1['a'][0] == 'a' def test_merge_list(): @@ -181,18 +180,18 @@ def test_merge_list(): dst1 = {} for d in [copy.deepcopy(src), copy.deepcopy(p1)]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [copy.deepcopy(src), copy.deepcopy(p2)]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1] dst1 = {} for d in [src, p1]: - dpath.util.merge(dst1, d) + dpath.merge(dst1, d) dst2 = {} for d in [src, p2]: - dpath.util.merge(dst2, d) + dpath.merge(dst2, d) assert dst1["l"] == [1, 2] assert dst2["l"] == [1, 2] diff --git a/tests/test_util_new.py b/tests/test_new.py similarity index 51% rename from tests/test_util_new.py rename to tests/test_new.py index d04b056..15b21c6 100644 --- a/tests/test_util_new.py +++ b/tests/test_new.py @@ -1,4 +1,4 @@ -import dpath.util +import dpath def test_set_new_separator(): @@ -7,11 +7,11 @@ def test_set_new_separator(): }, } - dpath.util.new(dict, ';a;b', 1, separator=";") - assert(dict['a']['b'] == 1) + dpath.new(dict, ';a;b', 1, separator=";") + assert dict['a']['b'] == 1 - dpath.util.new(dict, ['a', 'b'], 1, separator=";") - assert(dict['a']['b'] == 1) + dpath.new(dict, ['a', 'b'], 1, separator=";") + assert dict['a']['b'] == 1 def test_set_new_dict(): @@ -20,11 +20,11 @@ def test_set_new_dict(): }, } - dpath.util.new(dict, '/a/b', 1) - assert(dict['a']['b'] == 1) + dpath.new(dict, '/a/b', 1) + assert dict['a']['b'] == 1 - dpath.util.new(dict, ['a', 'b'], 1) - assert(dict['a']['b'] == 1) + dpath.new(dict, ['a', 'b'], 1) + assert dict['a']['b'] == 1 def test_set_new_list(): @@ -33,13 +33,23 @@ def test_set_new_list(): ], } - dpath.util.new(dict, '/a/1', 1) - assert(dict['a'][1] == 1) - assert(dict['a'][0] is None) + dpath.new(dict, '/a/1', 1) + assert dict['a'][1] == 1 + assert dict['a'][0] is None - dpath.util.new(dict, ['a', 1], 1) - assert(dict['a'][1] == 1) - assert(dict['a'][0] is None) + dpath.new(dict, ['a', 1], 1) + assert dict['a'][1] == 1 + assert dict['a'][0] is None + + +def test_set_list_with_dict_int_ambiguity(): + d = {"list": [{"root": {"1": {"k": None}}}]} + + dpath.new(d, "list/0/root/1/k", "new") + + expected = {"list": [{"root": {"1": {"k": "new"}}}]} + + assert d == expected def test_set_new_list_path_with_separator(): @@ -49,10 +59,10 @@ def test_set_new_list_path_with_separator(): }, } - dpath.util.new(dict, ['a', 'b/c/d', 0], 1) - assert(len(dict['a']) == 1) - assert(len(dict['a']['b/c/d']) == 1) - assert(dict['a']['b/c/d'][0] == 1) + dpath.new(dict, ['a', 'b/c/d', 0], 1) + assert len(dict['a']) == 1 + assert len(dict['a']['b/c/d']) == 1 + assert dict['a']['b/c/d'][0] == 1 def test_set_new_list_integer_path_with_creator(): @@ -76,8 +86,8 @@ def mycreator(obj, pathcomp, nextpathcomp, hints): obj[target] = {} print(obj) - dpath.util.new(d, '/a/2', 3, creator=mycreator) + dpath.new(d, '/a/2', 3, creator=mycreator) print(d) - assert(isinstance(d['a'], list)) - assert(len(d['a']) == 3) - assert(d['a'][2] == 3) + assert isinstance(d['a'], list) + assert len(d['a']) == 3 + assert d['a'][2] == 3 diff --git a/tests/test_path_get.py b/tests/test_path_get.py index 96347f2..b2a4657 100644 --- a/tests/test_path_get.py +++ b/tests/test_path_get.py @@ -15,6 +15,6 @@ def test_path_get_list_of_dicts(): segments = ['a', 'b', 0, 0] res = dpath.segments.view(tdict, segments) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 1) - assert(res['a']['b'][0][0] == 0) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 1 + assert res['a']['b'][0][0] == 0 diff --git a/tests/test_path_paths.py b/tests/test_path_paths.py index 3364e6d..56595d2 100644 --- a/tests/test_path_paths.py +++ b/tests/test_path_paths.py @@ -1,10 +1,10 @@ -from nose.tools import raises +from nose2.tools.such import helper + import dpath.segments import dpath.exceptions import dpath.options -@raises(dpath.exceptions.InvalidKeyName) def test_path_paths_empty_key_disallowed(): tdict = { "Empty": { @@ -14,8 +14,9 @@ def test_path_paths_empty_key_disallowed(): } } - for x in dpath.segments.walk(tdict): - pass + with helper.assertRaises(dpath.exceptions.InvalidKeyName): + for x in dpath.segments.walk(tdict): + pass def test_path_paths_empty_key_allowed(): @@ -34,4 +35,4 @@ def test_path_paths_empty_key_allowed(): pass dpath.options.ALLOW_EMPTY_STRING_KEYS = False - assert("/".join(segments) == "Empty//Key") + assert "/".join(segments) == "Empty//Key" diff --git a/tests/test_paths.py b/tests/test_paths.py new file mode 100644 index 0000000..c63d728 --- /dev/null +++ b/tests/test_paths.py @@ -0,0 +1,9 @@ +import dpath + + +def test_util_safe_path_list(): + res = dpath._split_path(["Ignore", "the/separator"], None) + + assert len(res) == 2 + assert res[0] == "Ignore" + assert res[1] == "the/separator" diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..5830088 --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,238 @@ +import dpath + + +def test_search_paths_with_separator(): + dict = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a', + 'a;b', + 'a;b;c', + 'a;b;c;d', + 'a;b;c;e', + 'a;b;c;f', + ] + + for (path, value) in dpath.search(dict, '/**', yielded=True, separator=";"): + assert path in paths + + for (path, value) in dpath.search(dict, ['**'], yielded=True, separator=";"): + assert path in paths + + +def test_search_paths(): + dict = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a', + 'a/b', + 'a/b/c', + 'a/b/c/d', + 'a/b/c/e', + 'a/b/c/f', + ] + + for (path, value) in dpath.search(dict, '/**', yielded=True): + assert path in paths + + for (path, value) in dpath.search(dict, ['**'], yielded=True): + assert path in paths + + +def test_search_afilter(): + def afilter(x): + if x in [1, 2]: + return True + return False + + dict = { + "a": { + "view_failure": "a", + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a/b/c/e', + 'a/b/c/f', + ] + + for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] + assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] + + for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): + assert path in paths + assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] + assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] + + +def test_search_globbing(): + dict = { + "a": { + "b": { + "c": { + "d": 0, + "e": 1, + "f": 2, + }, + }, + }, + } + paths = [ + 'a/b/c/d', + 'a/b/c/f', + ] + + for (path, value) in dpath.search(dict, '/a/**/[df]', yielded=True): + assert path in paths + + for (path, value) in dpath.search(dict, ['a', '**', '[df]'], yielded=True): + assert path in paths + + +def test_search_return_dict_head(): + tdict = { + "a": { + "b": { + 0: 0, + 1: 1, + 2: 2, + }, + }, + } + res = dpath.search(tdict, '/a/b') + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == {0: 0, 1: 1, 2: 2} + + res = dpath.search(tdict, ['a', 'b']) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == {0: 0, 1: 1, 2: 2} + + +def test_search_return_dict_globbed(): + tdict = { + "a": { + "b": { + 0: 0, + 1: 1, + 2: 2, + }, + }, + } + + res = dpath.search(tdict, '/a/b/[02]') + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 2 + assert res['a']['b'] == {0: 0, 2: 2} + + res = dpath.search(tdict, ['a', 'b', '[02]']) + assert isinstance(res['a']['b'], dict) + assert len(res['a']['b']) == 2 + assert res['a']['b'] == {0: 0, 2: 2} + + +def test_search_return_list_head(): + tdict = { + "a": { + "b": [ + 0, + 1, + 2, + ], + }, + } + + res = dpath.search(tdict, '/a/b') + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, 1, 2] + + res = dpath.search(tdict, ['a', 'b']) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, 1, 2] + + +def test_search_return_list_globbed(): + tdict = { + "a": { + "b": [ + 0, + 1, + 2, + ] + } + } + + res = dpath.search(tdict, '/a/b/[02]') + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, None, 2] + + res = dpath.search(tdict, ['a', 'b', '[02]']) + assert isinstance(res['a']['b'], list) + assert len(res['a']['b']) == 3 + assert res['a']['b'] == [0, None, 2] + + +def test_search_list_key_with_separator(): + tdict = { + "a": { + "b": { + "d": 'failure', + }, + "/b/d": 'success', + }, + } + + res = dpath.search(tdict, ['a', '/b/d']) + assert 'b' not in res['a'] + assert res['a']['/b/d'] == 'success' + + +def test_search_multiple_stars(): + testdata = { + 'a': [ + { + 'b': [ + {'c': 1}, + {'c': 2}, + {'c': 3}, + ], + }, + ], + } + testpath = 'a/*/b/*/c' + + res = dpath.search(testdata, testpath) + assert len(res['a'][0]['b']) == 3 + assert res['a'][0]['b'][0]['c'] == 1 + assert res['a'][0]['b'][1]['c'] == 2 + assert res['a'][0]['b'][2]['c'] == 3 diff --git a/tests/test_segments.py b/tests/test_segments.py index af9df85..fb6a8bc 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -1,8 +1,11 @@ -from dpath import options +import os +from unittest import TestCase + +import hypothesis.strategies as st from hypothesis import given, assume, settings, HealthCheck + import dpath.segments as api -import hypothesis.strategies as st -import os +from dpath import options settings.register_profile("default", suppress_health_check=(HealthCheck.too_slow,)) settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'default')) @@ -27,115 +30,6 @@ random_mutable_node = random_mutable_thing.filter(lambda thing: isinstance(thing, (list, dict))) -def setup(): - # Allow empty strings in segments. - options.ALLOW_EMPTY_STRING_KEYS = True - - -def teardown(): - # Revert back to default. - options.ALLOW_EMPTY_STRING_KEYS = False - - -@given(random_node) -def test_kvs(node): - ''' - Given a node, kvs should produce a key that when used to extract - from the node renders the exact same value given. - ''' - for k, v in api.kvs(node): - assert node[k] is v - - -@given(random_leaf) -def test_leaf_with_leaf(leaf): - ''' - Given a leaf, leaf should return True. - ''' - assert api.leaf(leaf) is True - - -@given(random_node) -def test_leaf_with_node(node): - ''' - Given a node, leaf should return False. - ''' - assert api.leaf(node) is False - - -@given(random_thing) -def test_walk(thing): - ''' - Given a thing to walk, walk should yield key, value pairs where key - is a tuple of non-zero length. - ''' - for k, v in api.walk(thing): - assert isinstance(k, tuple) - assert len(k) > 0 - - -@given(random_node) -def test_get(node): - ''' - Given a node, get should return the exact value given a key for all - key, value pairs in the node. - ''' - for k, v in api.walk(node): - assert api.get(node, k) is v - - -@given(random_node) -def test_has(node): - ''' - Given a node, has should return True for all paths, False otherwise. - ''' - for k, v in api.walk(node): - assert api.has(node, k) is True - - # If we are at a leaf, then we can create a value that isn't - # present easily. - if api.leaf(v): - assert api.has(node, k + (0,)) is False - - -@given(random_segments) -def test_expand(segments): - ''' - Given segments expand should produce as many results are there were - segments and the last result should equal the given segments. - ''' - count = len(segments) - result = list(api.expand(segments)) - - assert count == len(result) - - if count > 0: - assert segments == result[-1] - - -@given(random_node) -def test_types(node): - ''' - Given a node, types should yield a tuple of key, type pairs and the - type indicated should equal the type of the value. - ''' - for k, v in api.walk(node): - ts = api.types(node, k) - ta = () - for tk, tt in ts: - ta += (tk,) - assert type(api.get(node, ta)) is tt - - -@given(random_node) -def test_leaves(node): - ''' - Given a node, leaves should yield only leaf key, value pairs. - ''' - for k, v in api.leaves(node): - assert api.leafy(v) - - @st.composite def mutate(draw, segment): # Convert number segments. @@ -216,7 +110,7 @@ def random_segments_with_glob(draw): stop = draw(st.integers(start, len(glob))) glob[start:stop] = ['**'] - return (segments, glob) + return segments, glob @st.composite @@ -243,25 +137,6 @@ def random_segments_with_nonmatching_glob(draw): return (segments, glob) -@given(random_segments_with_glob()) -def test_match(pair): - ''' - Given segments and a known good glob, match should be True. - ''' - (segments, glob) = pair - assert api.match(segments, glob) is True - - -@given(random_segments_with_nonmatching_glob()) -def test_match_nonmatching(pair): - ''' - Given segments and a known bad glob, match should be False. - ''' - print(pair) - (segments, glob) = pair - assert api.match(segments, glob) is False - - @st.composite def random_walk(draw): node = draw(random_mutable_node) @@ -278,64 +153,179 @@ def random_leaves(draw): return (node, draw(st.sampled_from(found))) -@given(walkable=random_walk(), value=random_thing) -def test_set_walkable(walkable, value): - ''' - Given a walkable location, set should be able to update any value. - ''' - (node, (segments, found)) = walkable - api.set(node, segments, value) - assert api.get(node, segments) is value - - -@given(walkable=random_leaves(), - kstr=random_key_str, - kint=random_key_int, - value=random_thing, - extension=random_segments) -def test_set_create_missing(walkable, kstr, kint, value, extension): - ''' - Given a walkable non-leaf, set should be able to create missing - nodes and set a new value. - ''' - (node, (segments, found)) = walkable - assume(api.leaf(found)) - - parent_segments = segments[:-1] - parent = api.get(node, parent_segments) - - if isinstance(parent, list): - assume(len(parent) < kint) - destination = parent_segments + (kint,) + tuple(extension) - elif isinstance(parent, dict): - assume(kstr not in parent) - destination = parent_segments + (kstr,) + tuple(extension) - else: - raise Exception('mad mad world') - - api.set(node, destination, value) - assert api.get(node, destination) is value +class TestSegments(TestCase): + @classmethod + def setUpClass(cls): + # Allow empty strings in segments. + options.ALLOW_EMPTY_STRING_KEYS = True + + @classmethod + def tearDownClass(cls): + # Revert back to default. + options.ALLOW_EMPTY_STRING_KEYS = False + + @given(random_node) + def test_kvs(self, node): + ''' + Given a node, kvs should produce a key that when used to extract + from the node renders the exact same value given. + ''' + for k, v in api.make_walkable(node): + assert node[k] is v + + @given(random_leaf) + def test_leaf_with_leaf(self, leaf): + ''' + Given a leaf, leaf should return True. + ''' + assert api.leaf(leaf) is True + + @given(random_node) + def test_leaf_with_node(self, node): + ''' + Given a node, leaf should return False. + ''' + assert api.leaf(node) is False + + @given(random_thing) + def test_walk(self, thing): + ''' + Given a thing to walk, walk should yield key, value pairs where key + is a tuple of non-zero length. + ''' + for k, v in api.walk(thing): + assert isinstance(k, tuple) + assert len(k) > 0 + + @given(random_node) + def test_get(self, node): + ''' + Given a node, get should return the exact value given a key for all + key, value pairs in the node. + ''' + for k, v in api.walk(node): + assert api.get(node, k) is v + + @given(random_node) + def test_has(self, node): + ''' + Given a node, has should return True for all paths, False otherwise. + ''' + for k, v in api.walk(node): + assert api.has(node, k) is True + + # If we are at a leaf, then we can create a value that isn't + # present easily. + if api.leaf(v): + assert api.has(node, k + (0,)) is False + + @given(random_segments) + def test_expand(self, segments): + ''' + Given segments expand should produce as many results are there were + segments and the last result should equal the given segments. + ''' + count = len(segments) + result = list(api.expand(segments)) + + assert count == len(result) + + if count > 0: + assert segments == result[-1] + + @given(random_node) + def test_types(self, node): + ''' + Given a node, types should yield a tuple of key, type pairs and the + type indicated should equal the type of the value. + ''' + for k, v in api.walk(node): + ts = api.types(node, k) + ta = () + for tk, tt in ts: + ta += (tk,) + assert type(api.get(node, ta)) is tt + + @given(random_node) + def test_leaves(self, node): + ''' + Given a node, leaves should yield only leaf key, value pairs. + ''' + for k, v in api.leaves(node): + assert api.leafy(v) + + @given(random_segments_with_glob()) + def test_match(self, pair): + ''' + Given segments and a known good glob, match should be True. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is True + + @given(random_segments_with_nonmatching_glob()) + def test_match_nonmatching(self, pair): + ''' + Given segments and a known bad glob, match should be False. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is False + + @given(walkable=random_walk(), value=random_thing) + def test_set_walkable(self, walkable, value): + ''' + Given a walkable location, set should be able to update any value. + ''' + (node, (segments, found)) = walkable + api.set(node, segments, value) + assert api.get(node, segments) is value + + @given(walkable=random_leaves(), + kstr=random_key_str, + kint=random_key_int, + value=random_thing, + extension=random_segments) + def test_set_create_missing(self, walkable, kstr, kint, value, extension): + ''' + Given a walkable non-leaf, set should be able to create missing + nodes and set a new value. + ''' + (node, (segments, found)) = walkable + assume(api.leaf(found)) + + parent_segments = segments[:-1] + parent = api.get(node, parent_segments) + + if isinstance(parent, list): + assume(len(parent) < kint) + destination = parent_segments + (kint,) + tuple(extension) + elif isinstance(parent, dict): + assume(kstr not in parent) + destination = parent_segments + (kstr,) + tuple(extension) + else: + raise Exception('mad mad world') + api.set(node, destination, value) + assert api.get(node, destination) is value -@given(thing=random_thing) -def test_fold(thing): - ''' - Given a thing, count paths with fold. - ''' - def f(o, p, a): - a[0] += 1 + @given(thing=random_thing) + def test_fold(self, thing): + ''' + Given a thing, count paths with fold. + ''' - [count] = api.fold(thing, f, [0]) - assert count == len(tuple(api.walk(thing))) + def f(o, p, a): + a[0] += 1 + [count] = api.fold(thing, f, [0]) + assert count == len(tuple(api.walk(thing))) -@given(walkable=random_walk()) -def test_view(walkable): - ''' - Given a walkable location, view that location. - ''' - (node, (segments, found)) = walkable - assume(found == found) # Hello, nan! We don't want you here. + @given(walkable=random_walk()) + def test_view(self, walkable): + ''' + Given a walkable location, view that location. + ''' + (node, (segments, found)) = walkable + assume(found == found) # Hello, nan! We don't want you here. - view = api.view(node, segments) - assert api.get(view, segments) == api.get(node, segments) + view = api.view(node, segments) + assert api.get(view, segments) == api.get(node, segments) diff --git a/tests/test_set.py b/tests/test_set.py new file mode 100644 index 0000000..ef2dd96 --- /dev/null +++ b/tests/test_set.py @@ -0,0 +1,91 @@ +import dpath + + +def test_set_existing_separator(): + dict = { + "a": { + "b": 0, + }, + } + + dpath.set(dict, ';a;b', 1, separator=";") + assert dict['a']['b'] == 1 + + dict['a']['b'] = 0 + dpath.set(dict, ['a', 'b'], 1, separator=";") + assert dict['a']['b'] == 1 + + +def test_set_existing_dict(): + dict = { + "a": { + "b": 0, + }, + } + + dpath.set(dict, '/a/b', 1) + assert dict['a']['b'] == 1 + + dict['a']['b'] = 0 + dpath.set(dict, ['a', 'b'], 1) + assert dict['a']['b'] == 1 + + +def test_set_existing_list(): + dict = { + "a": [ + 0, + ], + } + + dpath.set(dict, '/a/0', 1) + assert dict['a'][0] == 1 + + dict['a'][0] = 0 + dpath.set(dict, ['a', '0'], 1) + assert dict['a'][0] == 1 + + +def test_set_filter(): + def afilter(x): + if int(x) == 31: + return True + return False + + dict = { + "a": { + "b": 0, + "c": 1, + "d": 31, + } + } + + dpath.set(dict, '/a/*', 31337, afilter=afilter) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert dict['a']['d'] == 31337 + + dict = { + "a": { + "b": 0, + "c": 1, + "d": 31, + } + } + + dpath.set(dict, ['a', '*'], 31337, afilter=afilter) + assert dict['a']['b'] == 0 + assert dict['a']['c'] == 1 + assert dict['a']['d'] == 31337 + + +def test_set_existing_path_with_separator(): + dict = { + "a": { + 'b/c/d': 0, + }, + } + + dpath.set(dict, ['a', 'b/c/d'], 1) + assert len(dict['a']) == 1 + assert dict['a']['b/c/d'] == 1 diff --git a/tests/test_types.py b/tests/test_types.py index 82f8c05..39993f3 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -1,19 +1,16 @@ -import nose -import dpath.util -from nose.tools import assert_raises +from collections.abc import MutableSequence, MutableMapping -try: - # python3, especially 3.8 - from collections.abc import MutableSequence - from collections.abc import MutableMapping -except ImportError: - # python2 - from collections import MutableSequence - from collections import MutableMapping +from nose2.tools.such import helper + +import dpath +from dpath import MergeType class TestMapping(MutableMapping): - def __init__(self, data={}): + def __init__(self, data=None): + if data is None: + data = {} + self._mapping = {} self._mapping.update(data) @@ -37,7 +34,10 @@ def __delitem__(self, key): class TestSequence(MutableSequence): - def __init__(self, data=list()): + def __init__(self, data=None): + if data is None: + data = list() + self._list = [] + data def __len__(self): @@ -71,13 +71,13 @@ def append(self, value): def test_types_set(): data = TestMapping({"a": TestSequence([0])}) - dpath.util.set(data, '/a/0', 1) - assert(data['a'][0] == 1) + dpath.set(data, '/a/0', 1) + assert data['a'][0] == 1 data['a'][0] = 0 - dpath.util.set(data, ['a', '0'], 1) - assert(data['a'][0] == 1) + dpath.set(data, ['a', '0'], 1) + assert data['a'][0] == 1 def test_types_get_list_of_dicts(): @@ -93,9 +93,9 @@ def test_types_get_list_of_dicts(): res = dpath.segments.view(tdict, ['a', 'b', 0, 0]) - assert(isinstance(res['a']['b'], TestSequence)) - assert(len(res['a']['b']) == 1) - assert(res['a']['b'][0][0] == 0) + assert isinstance(res['a']['b'], TestSequence) + assert len(res['a']['b']) == 1 + assert res['a']['b'][0][0] == 0 def test_types_merge_simple_list_replace(): @@ -106,14 +106,14 @@ def test_types_merge_simple_list_replace(): "list": TestSequence([0, 1, 2, 3]) }) - dpath.util.merge(dst, src, flags=dpath.util.MERGE_REPLACE) - nose.tools.eq_(dst["list"], TestSequence([7, 8, 9, 10])) + dpath.merge(dst, src, flags=MergeType.REPLACE) + assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) def test_types_get_absent(): ehash = TestMapping() - assert_raises(KeyError, dpath.util.get, ehash, '/a/b/c/d/f') - assert_raises(KeyError, dpath.util.get, ehash, ['a', 'b', 'c', 'd', 'f']) + helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') + helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) def test_types_get_glob_multiple(): @@ -130,8 +130,8 @@ def test_types_get_glob_multiple(): }), }) - assert_raises(ValueError, dpath.util.get, ehash, '/a/b/*/d') - assert_raises(ValueError, dpath.util.get, ehash, ['a', 'b', '*', 'd']) + helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') + helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) def test_delete_filter(): @@ -148,7 +148,7 @@ def afilter(x): }), }) - dpath.util.delete(data, '/a/*', afilter=afilter) - assert (data['a']['b'] == 0) - assert (data['a']['c'] == 1) - assert ('d' not in data['a']) + dpath.delete(data, '/a/*', afilter=afilter) + assert data['a']['b'] == 0 + assert data['a']['c'] == 1 + assert 'd' not in data['a'] diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 104e108..d4e8033 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,32 +1,32 @@ -import dpath.util +import dpath def test_unicode_merge(): a = {'中': 'zhong'} b = {'文': 'wen'} - dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a['中'] == 'zhong') - assert(a['文'] == 'wen') + dpath.merge(a, b) + assert len(a.keys()) == 2 + assert a['中'] == 'zhong' + assert a['文'] == 'wen' def test_unicode_search(): a = {'中': 'zhong'} - results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] - assert(len(results) == 1) - assert(results[0][0] == '中') - assert(results[0][1] == 'zhong') + results = [[x[0], x[1]] for x in dpath.search(a, '*', yielded=True)] + assert len(results) == 1 + assert results[0][0] == '中' + assert results[0][1] == 'zhong' def test_unicode_str_hybrid(): a = {'first': u'1'} b = {u'second': '2'} - dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a[u'second'] == '2') - assert(a['second'] == u'2') - assert(a[u'first'] == '1') - assert(a['first'] == u'1') + dpath.merge(a, b) + assert len(a.keys()) == 2 + assert a[u'second'] == '2' + assert a['second'] == u'2' + assert a[u'first'] == '1' + assert a['first'] == u'1' diff --git a/tests/test_util_paths.py b/tests/test_util_paths.py deleted file mode 100644 index 27260fe..0000000 --- a/tests/test_util_paths.py +++ /dev/null @@ -1,9 +0,0 @@ -import dpath.util - - -def test_util_safe_path_list(): - res = dpath.util.__safe_path__(["Ignore", "the/separator"], None) - - assert(len(res) == 2) - assert(res[0] == "Ignore") - assert(res[1] == "the/separator") diff --git a/tests/test_util_search.py b/tests/test_util_search.py deleted file mode 100644 index e7a4d43..0000000 --- a/tests/test_util_search.py +++ /dev/null @@ -1,238 +0,0 @@ -import dpath.util - - -def test_search_paths_with_separator(): - dict = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a', - 'a;b', - 'a;b;c', - 'a;b;c;d', - 'a;b;c;e', - 'a;b;c;f', - ] - - for (path, value) in dpath.util.search(dict, '/**', yielded=True, separator=";"): - assert(path in paths) - - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, separator=";"): - assert(path in paths) - - -def test_search_paths(): - dict = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a', - 'a/b', - 'a/b/c', - 'a/b/c/d', - 'a/b/c/e', - 'a/b/c/f', - ] - - for (path, value) in dpath.util.search(dict, '/**', yielded=True): - assert(path in paths) - - for (path, value) in dpath.util.search(dict, ['**'], yielded=True): - assert(path in paths) - - -def test_search_afilter(): - def afilter(x): - if x in [1, 2]: - return True - return False - - dict = { - "a": { - "view_failure": "a", - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a/b/c/e', - 'a/b/c/f', - ] - - for (path, value) in dpath.util.search(dict, '/**', yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, '/**', afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, '/**', afilter=afilter)['a']['b']['c']) - - for (path, value) in dpath.util.search(dict, ['**'], yielded=True, afilter=afilter): - assert(path in paths) - assert("view_failure" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']) - assert("d" not in dpath.util.search(dict, ['**'], afilter=afilter)['a']['b']['c']) - - -def test_search_globbing(): - dict = { - "a": { - "b": { - "c": { - "d": 0, - "e": 1, - "f": 2, - }, - }, - }, - } - paths = [ - 'a/b/c/d', - 'a/b/c/f', - ] - - for (path, value) in dpath.util.search(dict, '/a/**/[df]', yielded=True): - assert(path in paths) - - for (path, value) in dpath.util.search(dict, ['a', '**', '[df]'], yielded=True): - assert(path in paths) - - -def test_search_return_dict_head(): - tdict = { - "a": { - "b": { - 0: 0, - 1: 1, - 2: 2, - }, - }, - } - res = dpath.util.search(tdict, '/a/b') - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == {0: 0, 1: 1, 2: 2}) - - res = dpath.util.search(tdict, ['a', 'b']) - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == {0: 0, 1: 1, 2: 2}) - - -def test_search_return_dict_globbed(): - tdict = { - "a": { - "b": { - 0: 0, - 1: 1, - 2: 2, - }, - }, - } - - res = dpath.util.search(tdict, '/a/b/[02]') - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 2) - assert(res['a']['b'] == {0: 0, 2: 2}) - - res = dpath.util.search(tdict, ['a', 'b', '[02]']) - assert(isinstance(res['a']['b'], dict)) - assert(len(res['a']['b']) == 2) - assert(res['a']['b'] == {0: 0, 2: 2}) - - -def test_search_return_list_head(): - tdict = { - "a": { - "b": [ - 0, - 1, - 2, - ], - }, - } - - res = dpath.util.search(tdict, '/a/b') - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, 1, 2]) - - res = dpath.util.search(tdict, ['a', 'b']) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, 1, 2]) - - -def test_search_return_list_globbed(): - tdict = { - "a": { - "b": [ - 0, - 1, - 2, - ] - } - } - - res = dpath.util.search(tdict, '/a/b/[02]') - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, None, 2]) - - res = dpath.util.search(tdict, ['a', 'b', '[02]']) - assert(isinstance(res['a']['b'], list)) - assert(len(res['a']['b']) == 3) - assert(res['a']['b'] == [0, None, 2]) - - -def test_search_list_key_with_separator(): - tdict = { - "a": { - "b": { - "d": 'failure', - }, - "/b/d": 'success', - }, - } - - res = dpath.util.search(tdict, ['a', '/b/d']) - assert('b' not in res['a']) - assert(res['a']['/b/d'] == 'success') - - -def test_search_multiple_stars(): - testdata = { - 'a': [ - { - 'b': [ - {'c': 1}, - {'c': 2}, - {'c': 3}, - ], - }, - ], - } - testpath = 'a/*/b/*/c' - - res = dpath.util.search(testdata, testpath) - assert(len(res['a'][0]['b']) == 3) - assert(res['a'][0]['b'][0]['c'] == 1) - assert(res['a'][0]['b'][1]['c'] == 2) - assert(res['a'][0]['b'][2]['c'] == 3) diff --git a/tests/test_util_set.py b/tests/test_util_set.py deleted file mode 100644 index 3684a56..0000000 --- a/tests/test_util_set.py +++ /dev/null @@ -1,91 +0,0 @@ -import dpath.util - - -def test_set_existing_separator(): - dict = { - "a": { - "b": 0, - }, - } - - dpath.util.set(dict, ';a;b', 1, separator=";") - assert(dict['a']['b'] == 1) - - dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1, separator=";") - assert(dict['a']['b'] == 1) - - -def test_set_existing_dict(): - dict = { - "a": { - "b": 0, - }, - } - - dpath.util.set(dict, '/a/b', 1) - assert(dict['a']['b'] == 1) - - dict['a']['b'] = 0 - dpath.util.set(dict, ['a', 'b'], 1) - assert(dict['a']['b'] == 1) - - -def test_set_existing_list(): - dict = { - "a": [ - 0, - ], - } - - dpath.util.set(dict, '/a/0', 1) - assert(dict['a'][0] == 1) - - dict['a'][0] = 0 - dpath.util.set(dict, ['a', '0'], 1) - assert(dict['a'][0] == 1) - - -def test_set_filter(): - def afilter(x): - if int(x) == 31: - return True - return False - - dict = { - "a": { - "b": 0, - "c": 1, - "d": 31, - } - } - - dpath.util.set(dict, '/a/*', 31337, afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert (dict['a']['d'] == 31337) - - dict = { - "a": { - "b": 0, - "c": 1, - "d": 31, - } - } - - dpath.util.set(dict, ['a', '*'], 31337, afilter=afilter) - assert (dict['a']['b'] == 0) - assert (dict['a']['c'] == 1) - assert (dict['a']['d'] == 31337) - - -def test_set_existing_path_with_separator(): - dict = { - "a": { - 'b/c/d': 0, - }, - } - - dpath.util.set(dict, ['a', 'b/c/d'], 1) - assert(len(dict['a']) == 1) - assert(dict['a']['b/c/d'] == 1) diff --git a/tox.ini b/tox.ini index 5e521b2..d613837 100644 --- a/tox.ini +++ b/tox.ini @@ -7,23 +7,18 @@ ignore = E501,E722 [tox] -envlist = py36, pypy37, py38, py39, flake8 +envlist = pypy37, py38, py39, py310 [gh-actions] python = - 3.6: py36 pypy-3.7: pypy37 3.8: py38 - 3.9: py39, flake8 + 3.9: py39 + 3.10: py310 [testenv] deps = hypothesis mock - nose -commands = nosetests {posargs} - -[testenv:flake8] -deps = - flake8 -commands = flake8 setup.py dpath/ tests/ + nose2 +commands = nose2 {posargs} From 8adbce62d4ba750cc426f9a19aa05f22ac9ee6f1 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 18:35:34 +0200 Subject: [PATCH 04/23] Flesh out DDict class --- dpath/ddict.py | 87 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index c472620..4380a9d 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -1,6 +1,7 @@ from typing import Any, MutableMapping -from dpath import MergeType, merge, search, values, get, set, Filter, Glob +from dpath import Creator +from dpath.types import MergeType, Filter, Glob _DEFAULT_SENTINEL: Any = object() @@ -10,31 +11,83 @@ class DDict(dict): Glob aware dict """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, data: MutableMapping, separator="/", creator: Creator = None): + super().__init__(data) - def set(self, glob: Glob, value, separator="/", afilter: Filter = None): - return set(self, glob, value, separator=separator, afilter=afilter) + self.separator = separator + self.creator = creator - def get(self, glob: Glob, separator="/", default=_DEFAULT_SENTINEL) -> Any: + def __getitem__(self, item): + return self.get(item) + + def __contains__(self, item): + return len(self.search(item)) > 0 + + def __setitem__(self, key, value): + from dpath import new + + # Prevent infinite recursion and other issues + temp = dict(self) + + new(temp, key, value, separator=self.separator, creator=self.creator) + + self.update(temp) + + def __len__(self): + return len(self.keys()) + + def __or__(self, other): + return self.merge(other) + + def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: """ Same as dict.get but glob aware """ - if default is not _DEFAULT_SENTINEL: - # Default value was passed - return get(self, glob, separator=separator, default=default) - else: + from dpath import get + + if default is _DEFAULT_SENTINEL: # Let util.get handle default value - return get(self, glob, separator=separator) + return get(self, glob, separator=self.separator) + else: + # Default value was passed + return get(self, glob, separator=self.separator, default=default) + + # def keys(self): + # from dpath.segments import walk + # + # temp = dict(self) + # + # paths = (part[0] for part in walk(temp)) + # return (self.separator.join((str(segment) for segment in path)) for path in paths) - def values(self, glob: Glob = "*", separator="/", afilter: Filter = None, dirs=True): + def values(self, glob: Glob = "*", afilter: Filter = None, dirs=True): """ Same as dict.values but glob aware """ - return values(self, glob, separator=separator, afilter=afilter, dirs=dirs) + from dpath import values + + return values(self, glob, separator=self.separator, afilter=afilter, dirs=dirs) + + def search(self, glob: Glob, yielded=False, afilter: Filter = None, dirs=True): + from dpath import search + + return search(self, glob, yielded=yielded, separator=self.separator, afilter=afilter, dirs=dirs) + + def merge(self, src: MutableMapping, afilter: Filter = None, flags=MergeType.ADDITIVE): + from dpath import merge + + temp = dict(self) + + result = merge(temp, src, separator=self.separator, afilter=afilter, flags=flags) + + self.update(result) + + return self + + def items(self): + from dpath.segments import walk - def search(self, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): - return search(self, glob, yielded=yielded, separator=separator, afilter=afilter, dirs=dirs) + temp = dict(self) - def merge(self, src: MutableMapping, separator="/", afilter: Filter = None, flags=MergeType.ADDITIVE): - return merge(self, src, separator=separator, afilter=afilter, flags=flags) + for path, value in walk(temp): + yield self.separator.join((str(segment) for segment in path)), value From 7b4ac99c2afe97785518d6affd389643afb810de Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 18:35:44 +0200 Subject: [PATCH 05/23] Add initial OOP tests --- tests/test_oop.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/test_oop.py diff --git a/tests/test_oop.py b/tests/test_oop.py new file mode 100644 index 0000000..d8a7019 --- /dev/null +++ b/tests/test_oop.py @@ -0,0 +1,35 @@ +from dpath import DDict + + +def test_getitem(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + }) + + assert d["a"] == 1 + assert d["b/0"] == 12 + assert d["b/-1"] == 34 + + +def test_setitem(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + }) + + d["b/5"] = 1 + assert d["b"][5] == 1 + + d["c"] = [54, 43, 32, 21] + assert d["c"] == [54, 43, 32, 21] + + +def test_setitem_overwrite(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + }) + + d["b"] = "abc" + assert d["b"] == "abc" From b74bf005034949cca325e3d1a79d52f466eb5b3b Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 18:39:16 +0200 Subject: [PATCH 06/23] Remove negative index check --- tests/test_oop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oop.py b/tests/test_oop.py index d8a7019..79dca42 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -9,7 +9,7 @@ def test_getitem(): assert d["a"] == 1 assert d["b/0"] == 12 - assert d["b/-1"] == 34 + # assert d["b/-1"] == 34 def test_setitem(): From 9db7de2a97043c7ac4e1471fac06a7b7622473a4 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 18:45:47 +0200 Subject: [PATCH 07/23] Add merge operators --- dpath/ddict.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dpath/ddict.py b/dpath/ddict.py index 4380a9d..10433b9 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -1,3 +1,4 @@ +from copy import deepcopy from typing import Any, MutableMapping from dpath import Creator @@ -37,6 +38,12 @@ def __len__(self): return len(self.keys()) def __or__(self, other): + from dpath import merge + + copy = deepcopy(self) + return merge(copy, other, self.separator) + + def __ior__(self, other): return self.merge(other) def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: From 3df2d7ef4a4360778aa282b50454c180a65c7b4c Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 19:29:09 +0200 Subject: [PATCH 08/23] Add some OOP tests --- tests/test_oop.py | 103 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 3 deletions(-) diff --git a/tests/test_oop.py b/tests/test_oop.py index 79dca42..9b635ec 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -1,7 +1,9 @@ +from copy import deepcopy + from dpath import DDict -def test_getitem(): +def test_oop_getitem(): d = DDict({ "a": 1, "b": [12, 23, 34], @@ -12,7 +14,7 @@ def test_getitem(): # assert d["b/-1"] == 34 -def test_setitem(): +def test_oop_setitem(): d = DDict({ "a": 1, "b": [12, 23, 34], @@ -25,7 +27,7 @@ def test_setitem(): assert d["c"] == [54, 43, 32, 21] -def test_setitem_overwrite(): +def test_oop_setitem_overwrite(): d = DDict({ "a": 1, "b": [12, 23, 34], @@ -33,3 +35,98 @@ def test_setitem_overwrite(): d["b"] = "abc" assert d["b"] == "abc" + + +def test_oop_contains(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + assert "a" in d + assert "b" in d + assert "b/0" in d + assert "c/d/e/1" in d + + +def test_oop_len(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + }) + + assert len(d) == 5 + + +def test_oop_merge(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + expected_after = { + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + }, + "f": [54], + } + + before = deepcopy(d) + + assert d | {"f": [54]} == expected_after + + assert d == before + + d |= {"f": [54]} + + assert d != before + assert d == expected_after + + +def test_oop_keys(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + assert not set(d.keys()).difference({ + "a", + "b", + "c", + "b/0", + "b/1", + "b/2", + "c/d", + "c/d/e", + "c/d/e/0", + "c/d/e/1", + }) + + +def test_oop_values(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + }) + + assert list(d.values()) == [1, [12, 23, 34], 12, 23, 34] + From 3a881653e28ef58dd386088ff9383cd652045a75 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 19:55:33 +0200 Subject: [PATCH 09/23] Add delitem and walk methods --- dpath/ddict.py | 27 +++++++-------------------- tests/test_oop.py | 19 ------------------- 2 files changed, 7 insertions(+), 39 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index 10433b9..36133d1 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -34,6 +34,11 @@ def __setitem__(self, key, value): self.update(temp) + def __delitem__(self, key: Glob, afilter: Filter = None): + from dpath import delete + + delete(self, key, separator=self.separator, afilter=afilter) + def __len__(self): return len(self.keys()) @@ -59,22 +64,6 @@ def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: # Default value was passed return get(self, glob, separator=self.separator, default=default) - # def keys(self): - # from dpath.segments import walk - # - # temp = dict(self) - # - # paths = (part[0] for part in walk(temp)) - # return (self.separator.join((str(segment) for segment in path)) for path in paths) - - def values(self, glob: Glob = "*", afilter: Filter = None, dirs=True): - """ - Same as dict.values but glob aware - """ - from dpath import values - - return values(self, glob, separator=self.separator, afilter=afilter, dirs=dirs) - def search(self, glob: Glob, yielded=False, afilter: Filter = None, dirs=True): from dpath import search @@ -91,10 +80,8 @@ def merge(self, src: MutableMapping, afilter: Filter = None, flags=MergeType.ADD return self - def items(self): + def walk(self): from dpath.segments import walk - temp = dict(self) - - for path, value in walk(temp): + for path, value in walk(self): yield self.separator.join((str(segment) for segment in path)), value diff --git a/tests/test_oop.py b/tests/test_oop.py index 9b635ec..ae338a3 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -54,15 +54,6 @@ def test_oop_contains(): assert "c/d/e/1" in d -def test_oop_len(): - d = DDict({ - "a": 1, - "b": [12, 23, 34], - }) - - assert len(d) == 5 - - def test_oop_merge(): d = DDict({ "a": 1, @@ -120,13 +111,3 @@ def test_oop_keys(): "c/d/e/0", "c/d/e/1", }) - - -def test_oop_values(): - d = DDict({ - "a": 1, - "b": [12, 23, 34], - }) - - assert list(d.values()) == [1, [12, 23, 34], 12, 23, 34] - From 8b7662c813335ee71d0d4afe2eafa10771697654 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 21:15:32 +0200 Subject: [PATCH 10/23] Minor improvements --- dpath/ddict.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index 36133d1..3e5183d 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -53,7 +53,7 @@ def __ior__(self, other): def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: """ - Same as dict.get but glob aware + Same as dict.get but accepts glob aware keys. """ from dpath import get @@ -70,18 +70,21 @@ def search(self, glob: Glob, yielded=False, afilter: Filter = None, dirs=True): return search(self, glob, yielded=yielded, separator=self.separator, afilter=afilter, dirs=dirs) def merge(self, src: MutableMapping, afilter: Filter = None, flags=MergeType.ADDITIVE): + """ + Performs in-place merge with another dict. + """ from dpath import merge - temp = dict(self) - - result = merge(temp, src, separator=self.separator, afilter=afilter, flags=flags) + result = merge(self, src, separator=self.separator, afilter=afilter, flags=flags) self.update(result) return self def walk(self): + """ + Yields all possible key, value pairs. + """ from dpath.segments import walk - for path, value in walk(self): - yield self.separator.join((str(segment) for segment in path)), value + yield from ((self.separator.join((str(segment) for segment in path)), value) for path, value in walk(self)) From 252696cd7c787b298d72e252e2472b2cf814ab67 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 21:33:15 +0200 Subject: [PATCH 11/23] Add repr --- dpath/ddict.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpath/ddict.py b/dpath/ddict.py index 3e5183d..c1bc9fe 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -51,6 +51,9 @@ def __or__(self, other): def __ior__(self, other): return self.merge(other) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: """ Same as dict.get but accepts glob aware keys. From 2cf47858b67bdd2244ee8f677212c57baa38f952 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 21:48:14 +0200 Subject: [PATCH 12/23] Properly implement setitem and delitem --- dpath/ddict.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index c1bc9fe..3922fd7 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -12,11 +12,12 @@ class DDict(dict): Glob aware dict """ - def __init__(self, data: MutableMapping, separator="/", creator: Creator = None): - super().__init__(data) + def __init__(self, data: MutableMapping, __separator="/", __creator: Creator = None, **kwargs): + super().__init__() + super().update(data, **kwargs) - self.separator = separator - self.creator = creator + self.separator = __separator + self.creator = __creator def __getitem__(self, item): return self.get(item) @@ -32,12 +33,18 @@ def __setitem__(self, key, value): new(temp, key, value, separator=self.separator, creator=self.creator) + self.clear() self.update(temp) def __delitem__(self, key: Glob, afilter: Filter = None): from dpath import delete - delete(self, key, separator=self.separator, afilter=afilter) + temp = dict(self) + + delete(temp, key, separator=self.separator, afilter=afilter) + + self.clear() + self.update(temp) def __len__(self): return len(self.keys()) @@ -67,6 +74,11 @@ def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: # Default value was passed return get(self, glob, separator=self.separator, default=default) + def pop(self, __key: Glob): + results = self.search(__key) + del self[__key] + return results + def search(self, glob: Glob, yielded=False, afilter: Filter = None, dirs=True): from dpath import search From 9374f5861ce8555a4e2a29e8e7696653e1a77a4e Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 21:48:33 +0200 Subject: [PATCH 13/23] Add more tests --- tests/test_oop.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_oop.py b/tests/test_oop.py index ae338a3..a11a1ae 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -27,6 +27,54 @@ def test_oop_setitem(): assert d["c"] == [54, 43, 32, 21] +def test_oop_delitem(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + del d["a"] + assert "a" not in d + + del d["c/**/1"] + assert 67 not in d["c/d/e"] + + +def test_oop_pop(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + before = deepcopy(d) + + popped = d.pop("a") + assert popped == {"a": 1} + + d = deepcopy(before) + popped = d.pop("b/1") + assert popped == {"b": [None, 23]} + + d = deepcopy(before) + popped = d.pop("c/**/1") + assert popped == { + "c": { + "d": { + "e": [None, 67] + } + } + } + + def test_oop_setitem_overwrite(): d = DDict({ "a": 1, From 209e3544ebf570df6122728617ad036b4c48a36f Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 22:01:21 +0200 Subject: [PATCH 14/23] Implement pop and setdefault --- dpath/ddict.py | 21 ++++++++++++++------- tests/test_oop.py | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index 3922fd7..db851a9 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -25,23 +25,23 @@ def __getitem__(self, item): def __contains__(self, item): return len(self.search(item)) > 0 - def __setitem__(self, key, value): + def __setitem__(self, glob, value): from dpath import new # Prevent infinite recursion and other issues temp = dict(self) - new(temp, key, value, separator=self.separator, creator=self.creator) + new(temp, glob, value, separator=self.separator, creator=self.creator) self.clear() self.update(temp) - def __delitem__(self, key: Glob, afilter: Filter = None): + def __delitem__(self, glob: Glob, afilter: Filter = None): from dpath import delete temp = dict(self) - delete(temp, key, separator=self.separator, afilter=afilter) + delete(temp, glob, separator=self.separator, afilter=afilter) self.clear() self.update(temp) @@ -74,9 +74,16 @@ def get(self, glob: Glob, default=_DEFAULT_SENTINEL) -> Any: # Default value was passed return get(self, glob, separator=self.separator, default=default) - def pop(self, __key: Glob): - results = self.search(__key) - del self[__key] + def setdefault(self, glob: Glob, __default=_DEFAULT_SENTINEL): + if glob in self: + return self[glob] + + self[glob] = None if __default == _DEFAULT_SENTINEL else __default + return self[glob] + + def pop(self, glob: Glob): + results = self.search(glob) + del self[glob] return results def search(self, glob: Glob, yielded=False, afilter: Filter = None, dirs=True): diff --git a/tests/test_oop.py b/tests/test_oop.py index a11a1ae..02122fd 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -159,3 +159,26 @@ def test_oop_keys(): "c/d/e/0", "c/d/e/1", }) + + +def test_oop_setdefault(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + res = d.setdefault("a", 345) + assert res == 1 + + res = d.setdefault("c/4", 567) + assert res == 567 + assert d["c"]["4"] == res + + res = d.setdefault("b/6", 89) + assert res == 89 + assert d["b"] == [12, 23, 34, None, None, None, 89] From 03decc53a1c7b7270260d7e88acdfff7bf11b08c Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 22:05:07 +0200 Subject: [PATCH 15/23] Fix keys test --- tests/test_oop.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/test_oop.py b/tests/test_oop.py index 02122fd..61cdd35 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -147,18 +147,24 @@ def test_oop_keys(): } }) - assert not set(d.keys()).difference({ + # assert not { + # "a", + # "b", + # "c", + # "b/0", + # "b/1", + # "b/2", + # "c/d", + # "c/d/e", + # "c/d/e/0", + # "c/d/e/1", + # }.difference(set(d.keys())) + + assert not { "a", "b", "c", - "b/0", - "b/1", - "b/2", - "c/d", - "c/d/e", - "c/d/e/0", - "c/d/e/1", - }) + }.difference(set(d.keys())) def test_oop_setdefault(): From ba2df58bf78daa2cc736823b3611545bc9a01ca9 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 22:08:13 +0200 Subject: [PATCH 16/23] Remove len override --- dpath/ddict.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index db851a9..aa39073 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -45,10 +45,7 @@ def __delitem__(self, glob: Glob, afilter: Filter = None): self.clear() self.update(temp) - - def __len__(self): - return len(self.keys()) - + def __or__(self, other): from dpath import merge From ab7be787f0c957636eed4ac49a1718049eb0257d Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 22:48:51 +0200 Subject: [PATCH 17/23] Implement recursive keys, values and items --- dpath/ddict.py | 20 +++++++++++++++++++- tests/test_oop.py | 34 +++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index aa39073..8970cbb 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -19,6 +19,8 @@ def __init__(self, data: MutableMapping, __separator="/", __creator: Creator = N self.separator = __separator self.creator = __creator + self._recursive_items = True + def __getitem__(self, item): return self.get(item) @@ -45,7 +47,7 @@ def __delitem__(self, glob: Glob, afilter: Filter = None): self.clear() self.update(temp) - + def __or__(self, other): from dpath import merge @@ -100,6 +102,22 @@ def merge(self, src: MutableMapping, afilter: Filter = None, flags=MergeType.ADD return self + def keys(self): + for k, _ in self.walk(): + yield k + + def values(self): + for _, v in self.walk(): + yield v + + def items(self): + if not self._recursive_items: + yield from dict(self).items() + else: + self._recursive_items = False + yield from self.walk() + self._recursive_items = True + def walk(self): """ Yields all possible key, value pairs. diff --git a/tests/test_oop.py b/tests/test_oop.py index 61cdd35..91f422b 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -147,23 +147,17 @@ def test_oop_keys(): } }) - # assert not { - # "a", - # "b", - # "c", - # "b/0", - # "b/1", - # "b/2", - # "c/d", - # "c/d/e", - # "c/d/e/0", - # "c/d/e/1", - # }.difference(set(d.keys())) - assert not { "a", "b", "c", + "b/0", + "b/1", + "b/2", + "c/d", + "c/d/e", + "c/d/e/0", + "c/d/e/1", }.difference(set(d.keys())) @@ -188,3 +182,17 @@ def test_oop_setdefault(): res = d.setdefault("b/6", 89) assert res == 89 assert d["b"] == [12, 23, 34, None, None, None, 89] + + +def test_oop_items(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + assert len(list(d.items())) == 10 From 8fced03dcec99d3161d8e584c793ca282027bfbe Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 22:54:51 +0200 Subject: [PATCH 18/23] Expose option to control recursiveness of object --- dpath/ddict.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index 8970cbb..7d4d9f7 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -20,6 +20,7 @@ def __init__(self, data: MutableMapping, __separator="/", __creator: Creator = N self.creator = __creator self._recursive_items = True + self.recursive_items = True def __getitem__(self, item): return self.get(item) @@ -103,20 +104,29 @@ def merge(self, src: MutableMapping, afilter: Filter = None, flags=MergeType.ADD return self def keys(self): + if not self.recursive_items: + yield from super().keys() + return + for k, _ in self.walk(): yield k def values(self): + if not self.recursive_items: + yield from super().values() + return + for _, v in self.walk(): yield v def items(self): - if not self._recursive_items: + if not self.recursive_items or not self._recursive_items: yield from dict(self).items() - else: - self._recursive_items = False - yield from self.walk() - self._recursive_items = True + return + + self._recursive_items = False + yield from self.walk() + self._recursive_items = True def walk(self): """ From 1a630700550c8ee112185b190b831e77d5148731 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 4 Dec 2022 22:58:22 +0200 Subject: [PATCH 19/23] Reimplement len --- dpath/ddict.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dpath/ddict.py b/dpath/ddict.py index 7d4d9f7..edb5716 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -58,6 +58,9 @@ def __or__(self, other): def __ior__(self, other): return self.merge(other) + def __len__(self): + return sum(1 for _ in self.keys()) + def __repr__(self): return f"{type(self).__name__}({super().__repr__()})" From 6a2a788ec23bc4362b52a64cdb545e3a2497280c Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Mon, 5 Dec 2022 00:24:07 +0200 Subject: [PATCH 20/23] Rework keys and values --- dpath/ddict.py | 31 +++++++++++++------------------ tests/test_oop.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/dpath/ddict.py b/dpath/ddict.py index edb5716..adf16cc 100644 --- a/dpath/ddict.py +++ b/dpath/ddict.py @@ -19,7 +19,6 @@ def __init__(self, data: MutableMapping, __separator="/", __creator: Creator = N self.separator = __separator self.creator = __creator - self._recursive_items = True self.recursive_items = True def __getitem__(self, item): @@ -107,29 +106,24 @@ def merge(self, src: MutableMapping, afilter: Filter = None, flags=MergeType.ADD return self def keys(self): + from dpath.segments import walk + if not self.recursive_items: - yield from super().keys() + yield from dict(self).keys() return - for k, _ in self.walk(): - yield k + for path, _ in walk(self): + yield self.separator.join((str(segment) for segment in path)) def values(self): - if not self.recursive_items: - yield from super().values() - return - - for _, v in self.walk(): - yield v + from dpath.segments import walk - def items(self): - if not self.recursive_items or not self._recursive_items: - yield from dict(self).items() - return + d = self + if not self.recursive_items: + d = dict(self) - self._recursive_items = False - yield from self.walk() - self._recursive_items = True + for _, value in walk(d): + yield value def walk(self): """ @@ -137,4 +131,5 @@ def walk(self): """ from dpath.segments import walk - yield from ((self.separator.join((str(segment) for segment in path)), value) for path, value in walk(self)) + for path, value in walk(self): + yield self.separator.join((str(segment) for segment in path)), value diff --git a/tests/test_oop.py b/tests/test_oop.py index 91f422b..455ac98 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -136,6 +136,23 @@ def test_oop_merge(): assert d == expected_after +def test_oop_len(): + d = DDict({ + "a": 1, + "b": [12, 23, 34], + "c": { + "d": { + "e": [56, 67] + } + } + }) + + assert len(d) == 10, len(d) + + d.recursive_items = False + assert len(d) == 3, len(d) + + def test_oop_keys(): d = DDict({ "a": 1, From 3d7a021fcc36ab70fddd6a54dfb163401b4b6253 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Mon, 5 Dec 2022 00:24:14 +0200 Subject: [PATCH 21/23] Remove items test --- tests/test_oop.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/test_oop.py b/tests/test_oop.py index 455ac98..dc4c579 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -199,17 +199,3 @@ def test_oop_setdefault(): res = d.setdefault("b/6", 89) assert res == 89 assert d["b"] == [12, 23, 34, None, None, None, 89] - - -def test_oop_items(): - d = DDict({ - "a": 1, - "b": [12, 23, 34], - "c": { - "d": { - "e": [56, 67] - } - } - }) - - assert len(list(d.items())) == 10 From 3c90f752dc3559d5e00a2ca87160594bba0fe7e5 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Mon, 5 Dec 2022 09:55:02 +0200 Subject: [PATCH 22/23] Bump version --- dpath/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpath/version.py b/dpath/version.py index 5b0431e..3c00bb4 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.1" +VERSION = "2.2.0" From 106e56c887e901a12a47de7273bcc4b506c3e50b Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Mon, 5 Dec 2022 09:55:46 +0200 Subject: [PATCH 23/23] Enable negative index test --- tests/test_oop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oop.py b/tests/test_oop.py index dc4c579..201666e 100644 --- a/tests/test_oop.py +++ b/tests/test_oop.py @@ -11,7 +11,7 @@ def test_oop_getitem(): assert d["a"] == 1 assert d["b/0"] == 12 - # assert d["b/-1"] == 34 + assert d["b/-1"] == 34 def test_oop_setitem():