From dc04132be74f669e73d8ea9b082cb731df67f348 Mon Sep 17 00:00:00 2001 From: Aleksey Nekrasov Date: Fri, 24 Apr 2020 01:27:38 +0300 Subject: [PATCH 1/7] Added the ability to add your own marshmallow field mappings to the python type s s --- CHANGELOG.rst | 8 +++ combojsonapi/postgresql_jsonb/plugin.py | 77 +++++++++++++--------- tests/test_postgresql_jsonb/test_plugin.py | 53 ++++++++++++++- 3 files changed, 105 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a19f24b..cb9eea0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Changelog ********* +**Future** +========== + +Enhancements +============ + +* Added for custom marshmallow fields for PostgreSQL filtering (in PermissionPlugin) #- `@Znbiz`_ + **1.0.0** ========= diff --git a/combojsonapi/postgresql_jsonb/plugin.py b/combojsonapi/postgresql_jsonb/plugin.py index 08ebb79..fe5b29b 100644 --- a/combojsonapi/postgresql_jsonb/plugin.py +++ b/combojsonapi/postgresql_jsonb/plugin.py @@ -1,6 +1,6 @@ import datetime from decimal import Decimal -from typing import Any +from typing import Any, Optional, Union, Dict, Type import sqlalchemy from sqlalchemy import cast, String, Integer, Boolean, DECIMAL, not_ @@ -18,6 +18,13 @@ from combojsonapi.postgresql_jsonb.schema import SchemaJSONB +TYPE_MARSHMALLOW_FIELDS = Type[Union[ + ma_fields.Email, ma_fields.Dict, ma_fields.List, + ma_fields.Decimal, ma_fields.Url, ma_fields.DateTime, Any +]] +TYPE_PYTHON = Type[Union[str, dict, list, Decimal, datetime.datetime]] + + def is_seq_collection(obj): """ является ли переданный объект set, list, tuple @@ -28,6 +35,27 @@ def is_seq_collection(obj): class PostgreSqlJSONB(BasePlugin): + mapping_ma_field_to_type: Dict[TYPE_MARSHMALLOW_FIELDS, TYPE_PYTHON] = { + ma_fields.Email: str, + ma_fields.Dict: dict, + ma_fields.List: list, + ma_fields.Decimal: Decimal, + ma_fields.Url: str, + ma_fields.DateTime: datetime.datetime, + } + + def get_property_type( + self, marshmallow_field: TYPE_MARSHMALLOW_FIELDS, schema: Optional[Schema] = None + ) -> TYPE_PYTHON: + if schema is not None: + self.mapping_ma_field_to_type.update({ + v: k for k, v in schema.TYPE_MAPPING.items() + }) + return self.mapping_ma_field_to_type[type(marshmallow_field)] + + def add_mapping_field_to_python_type(self, marshmallow_field: Any, type_python: TYPE_PYTHON) -> None: + self.mapping_ma_field_to_type[marshmallow_field] = type_python + def before_data_layers_sorting_alchemy_nested_resolve(self, self_nested: Any) -> Any: """ Вызывается до создания сортировки в функции Nested.resolve, если после выполнения вернёт None, то @@ -86,8 +114,7 @@ def _isinstance_jsonb(cls, schema: Schema, filter_name): return False return False - @classmethod - def _create_sort(cls, self_nested: Any, marshmallow_field, model_column, order): + def _create_sort(self, self_nested: Any, marshmallow_field, model_column, order): """ Create sqlalchemy sort :param Nested self_nested: @@ -106,7 +133,7 @@ def _create_sort(cls, self_nested: Any, marshmallow_field, model_column, order): self_nested.sort_["field"] = SPLIT_REL.join(fields[1:]) marshmallow_field = marshmallow_field.schema._declared_fields[fields[1]] model_column = getattr(mapper, sqlalchemy_relationship_name) - return cls._create_sort(self_nested, marshmallow_field, model_column, order) + return self._create_sort(self_nested, marshmallow_field, model_column, order) elif not isinstance(getattr(marshmallow_field, "schema", None), SchemaJSONB): raise InvalidFilters(f"Invalid JSONB sort: {SPLIT_REL.join(self_nested.fields)}") fields = self_nested.sort_["field"].split(SPLIT_REL) @@ -126,16 +153,10 @@ def _create_sort(cls, self_nested: Any, marshmallow_field, model_column, order): return getattr(marshmallow_field, f"_{order}_sql_filter_")( marshmallow_field=marshmallow_field, model_column=model_column ) - mapping_ma_field_to_type = {v: k for k, v in self_nested.schema.TYPE_MAPPING.items()} - mapping_ma_field_to_type[ma_fields.Email] = str - mapping_ma_field_to_type[ma_fields.Dict] = dict - mapping_ma_field_to_type[ma_fields.List] = list - mapping_ma_field_to_type[ma_fields.Decimal] = Decimal - mapping_ma_field_to_type[ma_fields.Url] = str - mapping_ma_field_to_type[ma_fields.DateTime] = datetime.datetime + + property_type = self.get_property_type(marshmallow_field=marshmallow_field, schema=self_nested.schema) mapping_type_to_sql_type = {str: String, bytes: String, Decimal: DECIMAL, int: Integer, bool: Boolean} - property_type = mapping_ma_field_to_type[type(marshmallow_field)] extra_field = model_column.op("->>")(field_in_jsonb) sort = "" order_op = desc_op if order == "desc" else asc_op @@ -146,8 +167,7 @@ def _create_sort(cls, self_nested: Any, marshmallow_field, model_column, order): sort = order_op(extra_field.cast(mapping_type_to_sql_type[property_type])) return sort - @classmethod - def _create_filter(cls, self_nested: Any, marshmallow_field, model_column, operator, value): + def _create_filter(self, self_nested: Any, marshmallow_field, model_column, operator, value): """ Create sqlalchemy filter :param Nested self_nested: @@ -168,7 +188,7 @@ def _create_filter(cls, self_nested: Any, marshmallow_field, model_column, opera marshmallow_field = marshmallow_field.schema._declared_fields[fields[1]] join_list = [[model_column]] model_column = getattr(mapper, sqlalchemy_relationship_name) - filter, joins = cls._create_filter(self_nested, marshmallow_field, model_column, operator, value) + filter, joins = self._create_filter(self_nested, marshmallow_field, model_column, operator, value) join_list += joins return filter, join_list elif not isinstance(getattr(marshmallow_field, "schema", None), SchemaJSONB): @@ -198,40 +218,33 @@ def _create_filter(cls, self_nested: Any, marshmallow_field, model_column, opera ), [], ) - mapping = {v: k for k, v in self_nested.schema.TYPE_MAPPING.items()} - mapping[ma_fields.Email] = str - mapping[ma_fields.Dict] = dict - mapping[ma_fields.List] = list - mapping[ma_fields.Decimal] = Decimal - mapping[ma_fields.Url] = str - mapping[ma_fields.DateTime] = datetime.datetime # Нужно проводить валидацию и делать десериализацию значение указанных в фильтре, так как поля Enum # например выгружаются как 'name_value(str)', а в БД хранится как просто число value = deserialize_field(marshmallow_field, value) - property_type = mapping[type(marshmallow_field)] + property_type = self.get_property_type(marshmallow_field=marshmallow_field, schema=self_nested.schema) extra_field = model_column.op("->>")(field_in_jsonb) - filter = "" + filter_ = "" if property_type == Decimal: - filter = getattr(cast(extra_field, DECIMAL), self_nested.operator)(value) + filter_ = getattr(cast(extra_field, DECIMAL), self_nested.operator)(value) if property_type in {str, bytes}: - filter = getattr(cast(extra_field, String), self_nested.operator)(value) + filter_ = getattr(cast(extra_field, String), self_nested.operator)(value) if property_type == int: field = cast(extra_field, Integer) if value: - filter = getattr(field, self_nested.operator)(value) + filter_ = getattr(field, self_nested.operator)(value) else: - filter = or_(getattr(field, self_nested.operator)(value), field.is_(None)) + filter_ = or_(getattr(field, self_nested.operator)(value), field.is_(None)) if property_type == bool: - filter = cast(extra_field, Boolean) == value + filter_ = cast(extra_field, Boolean) == value if property_type == list: - filter = model_column.op("->")(field_in_jsonb).op("?")(value[0] if is_seq_collection(value) else value) + filter_ = model_column.op("->")(field_in_jsonb).op("?")(value[0] if is_seq_collection(value) else value) if operator in ["notin", "notin_"]: - filter = not_(filter) + filter_ = not_(filter_) - return filter, [] + return filter_, [] diff --git a/tests/test_postgresql_jsonb/test_plugin.py b/tests/test_postgresql_jsonb/test_plugin.py index c2efa67..4ec1922 100644 --- a/tests/test_postgresql_jsonb/test_plugin.py +++ b/tests/test_postgresql_jsonb/test_plugin.py @@ -1,3 +1,5 @@ +from typing import List, Optional +from unittest import mock from unittest.mock import Mock import pytest @@ -14,9 +16,31 @@ def plugin(): @pytest.fixture -def schema(): +def custom_field(): + class CustomEnumField(fields.Integer): + + def __init__(self, *args, default: int = 0, allowed_values: Optional[List[int]] = None, **kwargs): + self.default = default + self.allowed_values = allowed_values or [] + super().__init__(*args, enum=[1, 2, 3, 4], **kwargs) + + def _deserialize(self, value, attr, data, **kwargs): + try: + value = int(value) + value = value if value in self.allowed_values else self.default + except TypeError as e: + value = self.default + return value + + return CustomEnumField + + +@pytest.fixture +def schema(custom_field): + class TestSchema(SchemaJSONB): name = fields.Integer() + type_test = custom_field(allowed_values=[1, 2, 3, 5, 8]) class ParentSchema(Schema): test_schema = fields.Nested('TestSchema') @@ -36,3 +60,30 @@ def test_before_data_layers_sorting_alchemy_nested_resolve(self, plugin, schema) res = plugin.before_data_layers_sorting_alchemy_nested_resolve(mock_self_nested) assert res == (True, [],) + + @mock.patch('combojsonapi.postgresql_jsonb.plugin.cast', autospec=True) + def test_custom_mapping(self, mock_cast, plugin, schema, custom_field): + mock_self_nested = Mock() + mock_operator = 'eq' + mock_value = 1 + mock_self_nested.filter_ = { + 'name': f'test_schema{SPLIT_REL}type_test', + 'op': mock_operator, + 'val': mock_value, + } + mock_self_nested.schema = schema() + mock_self_nested.operator = mock_operator + mock_marshmallow_field = schema().fields['test_schema'] + mock_model_column = Mock() + plugin.add_mapping_field_to_python_type(custom_field, int) + mock_cast.eq = Mock(return_value=True) + + assert True, [] == plugin._create_filter( + self_nested=mock_self_nested, + marshmallow_field=mock_marshmallow_field, + model_column=mock_model_column, + operator=mock_operator, + value=mock_value + ) + + From 256f1803d46b76f62c59a1a73fcb4a6b213113f1 Mon Sep 17 00:00:00 2001 From: Dmitry Tarasov Date: Wed, 22 Jul 2020 14:14:28 +0300 Subject: [PATCH 2/7] 16-add-search-and-sort-by-sublvl-jsonb --- combojsonapi/postgresql_jsonb/plugin.py | 22 ++++- tests/test_postgresql_jsonb/test_plugin.py | 99 ++++++++++++++++++++++ 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/combojsonapi/postgresql_jsonb/plugin.py b/combojsonapi/postgresql_jsonb/plugin.py index fe5b29b..a3eedb1 100644 --- a/combojsonapi/postgresql_jsonb/plugin.py +++ b/combojsonapi/postgresql_jsonb/plugin.py @@ -105,7 +105,7 @@ def _isinstance_jsonb(cls, schema: Schema, filter_name): fields = filter_name.split(SPLIT_REL) for i, i_field in enumerate(fields): if isinstance(getattr(schema._declared_fields[i_field], "schema", None), SchemaJSONB): - if i != (len(fields) - 2): + if i == (len(fields) - 1): raise InvalidFilters(f"Invalid JSONB filter: {filter_name}") return True elif isinstance(schema._declared_fields[i_field], Relationship): @@ -140,7 +140,8 @@ def _create_sort(self, self_nested: Any, marshmallow_field, model_column, order) self_nested.sort_["field"] = SPLIT_REL.join(fields[:-1]) field_in_jsonb = fields[-1] - marshmallow_field = marshmallow_field.schema._declared_fields[field_in_jsonb] + for field in fields[1:]: + marshmallow_field = marshmallow_field.schema._declared_fields[field] if hasattr(marshmallow_field, f"_{order}_sql_filter_"): """ У marshmallow field может быть реализована своя логика создания сортировки для sqlalchemy @@ -150,6 +151,11 @@ def _create_sort(self, self_nested: Any, marshmallow_field, model_column, order) * marshmallow_field - объект класса поля marshmallow * model_column - объект класса поля sqlalchemy """ + # All values between the first and last field will be the path to the desired value by which to sort, + # so we write the path through "->" + for field in fields[1:-1]: + model_column = model_column.op("->")(field) + model_column = model_column.op("->>")(field_in_jsonb) return getattr(marshmallow_field, f"_{order}_sql_filter_")( marshmallow_field=marshmallow_field, model_column=model_column ) @@ -157,6 +163,8 @@ def _create_sort(self, self_nested: Any, marshmallow_field, model_column, order) property_type = self.get_property_type(marshmallow_field=marshmallow_field, schema=self_nested.schema) mapping_type_to_sql_type = {str: String, bytes: String, Decimal: DECIMAL, int: Integer, bool: Boolean} + for field in fields[1:-1]: + model_column = model_column.op("->")(field) extra_field = model_column.op("->>")(field_in_jsonb) sort = "" order_op = desc_op if order == "desc" else asc_op @@ -195,7 +203,8 @@ def _create_filter(self, self_nested: Any, marshmallow_field, model_column, oper raise InvalidFilters(f"Invalid JSONB filter: {SPLIT_REL.join(field_in_jsonb)}") self_nested.filter_["name"] = SPLIT_REL.join(fields[:-1]) try: - marshmallow_field = marshmallow_field.schema._declared_fields[field_in_jsonb] + for field in fields[1:]: + marshmallow_field = marshmallow_field.schema._declared_fields[field] except KeyError: raise InvalidFilters(f'There is no "{field_in_jsonb}" attribute in the "{fields[-2]}" field.') if hasattr(marshmallow_field, f"_{operator}_sql_filter_"): @@ -209,10 +218,13 @@ def _create_filter(self, self_nested: Any, marshmallow_field, model_column, oper * value - значения для фильтра * operator - сам оператор, например: "eq", "in"... """ + for field in fields[1:-1]: + model_column = model_column.op("->")(field) + model_column = model_column.op("->>")(field_in_jsonb) return ( getattr(marshmallow_field, f"_{operator}_sql_filter_")( marshmallow_field=marshmallow_field, - model_column=model_column.op("->>")(field_in_jsonb), + model_column=model_column, value=value, operator=self_nested.operator, ), @@ -224,6 +236,8 @@ def _create_filter(self, self_nested: Any, marshmallow_field, model_column, oper value = deserialize_field(marshmallow_field, value) property_type = self.get_property_type(marshmallow_field=marshmallow_field, schema=self_nested.schema) + for field in fields[1:-1]: + model_column = model_column.op("->")(field) extra_field = model_column.op("->>")(field_in_jsonb) filter_ = "" if property_type == Decimal: diff --git a/tests/test_postgresql_jsonb/test_plugin.py b/tests/test_postgresql_jsonb/test_plugin.py index 4ec1922..857a85a 100644 --- a/tests/test_postgresql_jsonb/test_plugin.py +++ b/tests/test_postgresql_jsonb/test_plugin.py @@ -40,14 +40,30 @@ def schema(custom_field): class TestSchema(SchemaJSONB): name = fields.Integer() + lvl1 = fields.Nested('TestSchemaLvl2') type_test = custom_field(allowed_values=[1, 2, 3, 5, 8]) + class TestSchemaLvl2(SchemaJSONB): + name = fields.String() + list = fields.List(fields.String()) + list._ilike_sql_filter_ = assert_custom_opertor + name._desc_sql_filter_ = assert_custom_sort + class ParentSchema(Schema): test_schema = fields.Nested('TestSchema') return ParentSchema +def assert_custom_opertor(marshmallow_field, model_column, value, operator): + assert operator == '__ilike__' + return True + + +def assert_custom_sort(marshmallow_field, model_column): + return True + + class TestPostgreSqlJSONB: def test_before_data_layers_sorting_alchemy_nested_resolve(self, plugin, schema): mock_self_nested = Mock() @@ -86,4 +102,87 @@ def test_custom_mapping(self, mock_cast, plugin, schema, custom_field): value=mock_value ) + def test__create_sort(self, plugin, schema): + mock_self_nested = Mock() + mock_self_nested.sort_ = {'field': f'test_schema{SPLIT_REL}lvl1{SPLIT_REL}name', 'order': 'asc'} + mock_self_nested.name = 'test_schema' + mock_self_nested.schema = schema + mock_marshmallow_field = schema().fields['test_schema'] + mock_model_column = Mock() + + plugin._create_sort( + self_nested=mock_self_nested, + marshmallow_field=mock_marshmallow_field, + model_column=mock_model_column, + order='desc') + + mock_model_column.op("->").assert_called_once() + + def test__create_sort_with_custom_sort(self, plugin, schema): + mock_self_nested = Mock() + mock_self_nested.sort_ = {'field': f'test_schema{SPLIT_REL}lvl1{SPLIT_REL}name', 'order': 'decs'} + mock_self_nested.name = 'test_schema' + mock_self_nested.schema = schema + mock_marshmallow_field = schema().fields['test_schema'] + mock_model_column = Mock() + + res = plugin._create_sort( + self_nested=mock_self_nested, + marshmallow_field=mock_marshmallow_field, + model_column=mock_model_column, + order='desc') + + mock_model_column.op("->").assert_called_once() + assert res == True + + def test__create_filter(self, plugin, schema): + mock_operator = 'eq' + mock_value = 'string' + mock_self_nested = Mock() + mock_self_nested.filter_ = { + 'name': f'test_schema{SPLIT_REL}lvl1{SPLIT_REL}name', + 'op': mock_operator, + 'val': mock_value, + } + mock_self_nested.operator = '__eq__' + mock_self_nested.name = 'test_schema' + mock_self_nested.name = 'test_schema' + + mock_self_nested.schema = schema + mock_marshmallow_field = schema().fields['test_schema'] + mock_model_column = Mock() + + + plugin._create_filter( + self_nested=mock_self_nested, + marshmallow_field=mock_marshmallow_field, + model_column=mock_model_column, + operator=mock_operator, + value=mock_value) + + mock_model_column.op("->").assert_called_once() + + def test__create_filter_with_custom_op(self, plugin, schema): + mock_operator = 'ilike' + mock_value = 'string' + mock_self_nested = Mock() + mock_self_nested.filter_ = { + 'name': f'test_schema{SPLIT_REL}lvl1{SPLIT_REL}list', + 'op': mock_operator, + 'val': mock_value, + } + mock_self_nested.operator = '__ilike__' + mock_self_nested.name = 'test_schema' + + mock_self_nested.schema = schema + mock_marshmallow_field = schema().fields['test_schema'] + mock_model_column = Mock() + + plugin._create_filter( + self_nested=mock_self_nested, + marshmallow_field=mock_marshmallow_field, + model_column=mock_model_column, + operator=mock_operator, + value=mock_value) + mock_model_column.op("->").assert_called_once() From 8fae29f6398eba3db335fd8453822fef60715574 Mon Sep 17 00:00:00 2001 From: Suren Khorenyan Date: Wed, 29 Jul 2020 18:25:02 +0300 Subject: [PATCH 3/7] Changelog for v1.0.3 --- CHANGELOG.rst | 12 +++++++++++- setup.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cb9eea0..c431113 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,15 @@ Enhancements * Added for custom marshmallow fields for PostgreSQL filtering (in PermissionPlugin) #- `@Znbiz`_ +**1.0.3** +========= + +Changes +======= + +* Filtering and sorting nested JSONB fields #- `@tarasovdg1`_ + + **1.0.0** ========= @@ -69,4 +78,5 @@ Enhancements .. _`@mahenzon`: https://github.com/mahenzon .. _`@Znbiz`: https://github.com/znbiz -.. _`@Yakov Shapovalov`: https://github.com/photovirus \ No newline at end of file +.. _`@Yakov Shapovalov`: https://github.com/photovirus +.. _`@tarasovdg1`: https://github.com/tarasovdg1 diff --git a/setup.py b/setup.py index 6b59050..ee3086e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -__version__ = "1.0.2" +__version__ = "1.0.3" setup( name="ComboJSONAPI", From c98bb5799274ba9e2e23b40961624daf20ae0b5e Mon Sep 17 00:00:00 2001 From: Suren Khorenyan Date: Wed, 29 Jul 2020 18:25:02 +0300 Subject: [PATCH 4/7] Changelog fixes --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c431113..18471a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,10 +4,15 @@ Changelog **Future** ========== + +**1.0.3** +========= + Enhancements ============ * Added for custom marshmallow fields for PostgreSQL filtering (in PermissionPlugin) #- `@Znbiz`_ +* Filtering and sorting nested JSONB fields (PostgreSqlJSONB) #- `@tarasovdg1`_ **1.0.3** From 835d0126abf466e3ed835aa85dc15941640ee20a Mon Sep 17 00:00:00 2001 From: Suren Khorenyan Date: Wed, 29 Jul 2020 18:30:14 +0300 Subject: [PATCH 5/7] Changelog one more fix --- CHANGELOG.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 18471a8..6437b9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,9 +1,6 @@ Changelog ********* -**Future** -========== - **1.0.3** ========= From 61672aa76cd95cd1d8ab5dfe09f0dd4e655f709d Mon Sep 17 00:00:00 2001 From: Suren Khorenyan Date: Wed, 29 Jul 2020 18:31:53 +0300 Subject: [PATCH 6/7] Changelog hopefully last fix --- CHANGELOG.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6437b9b..c9dcf8d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,15 +12,6 @@ Enhancements * Filtering and sorting nested JSONB fields (PostgreSqlJSONB) #- `@tarasovdg1`_ -**1.0.3** -========= - -Changes -======= - -* Filtering and sorting nested JSONB fields #- `@tarasovdg1`_ - - **1.0.0** ========= From e233a33a316754e5106b40e7c13eec139ea12b5b Mon Sep 17 00:00:00 2001 From: Suren Khorenyan Date: Wed, 29 Jul 2020 18:35:25 +0300 Subject: [PATCH 7/7] OK now it's definitely the last fix in the changelog for v1.0.3 --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c9dcf8d..6d430f2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,8 +8,8 @@ Changelog Enhancements ============ -* Added for custom marshmallow fields for PostgreSQL filtering (in PermissionPlugin) #- `@Znbiz`_ -* Filtering and sorting nested JSONB fields (PostgreSqlJSONB) #- `@tarasovdg1`_ +* Add custom marshmallow fields for PostgreSQL filtering (PostgreSqlJSONB plugin) #- `@Znbiz`_ +* Filtering and sorting nested JSONB fields (PostgreSqlJSONB plugin) #- `@tarasovdg1`_ **1.0.0**