From 4b64711fad8956b738c29f75c8d2286f8b576539 Mon Sep 17 00:00:00 2001 From: Paul Balluff Date: Wed, 24 Aug 2022 20:57:49 +0200 Subject: [PATCH] #175 #177 #178 --- flaskinventory/flaskdgraph/dgraph_types.py | 192 ++++++++++++++++-- flaskinventory/flaskdgraph/query.py | 64 +++--- flaskinventory/flaskdgraph/schema.py | 20 +- flaskinventory/main/model.py | 22 +- .../templates/helpers/_formhelpers.html | 2 +- flaskinventory/templates/query/sidebar.html | 82 +++++++- flaskinventory/templates/query/tomselect.html | 2 +- flaskinventory/view/routes.py | 6 +- tests/test_queries.py | 5 +- 9 files changed, 318 insertions(+), 77 deletions(-) diff --git a/flaskinventory/flaskdgraph/dgraph_types.py b/flaskinventory/flaskdgraph/dgraph_types.py index 7dd8ee66..968565fd 100644 --- a/flaskinventory/flaskdgraph/dgraph_types.py +++ b/flaskinventory/flaskdgraph/dgraph_types.py @@ -95,6 +95,150 @@ def update_facets(self, facets: dict) -> None: def nquad(self) -> str: return f'{self.newid}' +class Facet: + """ + Base class for facets. Use simple coercion. + Facet keys are strings and values can be string, bool, int, float and dateTime. + """ + + predicate = None + default_operator = "eq" + + def __init__(self, key: str, + dtype: Union[str, bool, int, float, datetime.datetime]=str, + queryable=False, + coerce=None, + query_label=None, + comparison_operators=None, + render_kw=None, + choices=None) -> None: + + self.key = key + self.type = dtype + self.queryable = queryable + self._query_label = query_label + if isinstance(comparison_operators, dict): + comparison_operators = [(k, v) for k, v in comparison_operators.items()] + comparison_operators.insert(0, ('','')) + self.operators = comparison_operators + self.render_kw = render_kw or {} + + if isinstance(choices, dict): + choices = [(k, v) for k, v in choices.items()] + + self.choices = choices + + def __repr__(self) -> str: + if self.predicate: + return f'<{self.type.__name__} Facet "{self.key}" of "{self.predicate}">' + else: + return f'' + + def __str__(self) -> str: + if self.predicate: + return f'{self.predicate}|{self.key}' + else: + return f'{self.key}' + + def corece(self, val) -> Any: + if self.type == bool: + return self._coerce_bool(val) + elif self.type == datetime.datetime: + try: + return self._coerce_datetime(val) + except: + return None + else: + try: + return self.type(val) + except: + return None + + def query_filter(self, vals: Union[str, list], operator=None, predicate=None, **kwargs) -> str: + + if not predicate: + predicate = self.predicate + + if not operator: + operator = self.default_operator + + if vals is None: + return None + + if not isinstance(vals, list): + vals = [vals] + + + if len(vals) == 0: + return None + + if self.type == datetime.datetime: + vals = [self.corece(vals)] + + if operator == 'between': + if self.type == datetime.datetime: + return f'ge({self.key}, "{vals[0].strftime("%Y-%m-%d")}") AND lt({self.key}, "{vals[1].strftime("%Y-%m-%d")}")' + else: + return f'ge({self.key}, "{self.coerce(vals[0])}") AND lt({self.key}, "{self.corece(vals[1])}")' + + else: + if self.type == datetime.datetime: + val1 = vals[0] + val2 = val1 + datetime.timedelta(days=1) + return f'ge({self.key}, "{val1.strftime("%Y-%m-%d")}") AND lt({self.key}, "{val2.strftime("%Y-%m-%d")}")' + filters = [f'{operator}({self.key}, "{self.corece(val)}")' for val in vals] + if len(filters) > 1: + filter_string = " OR ".join(filters) + return f'({filter_string})' + else: + return filters[0] + + + @staticmethod + def _coerce_bool(val) -> bool: + if isinstance(val, bool): + return bool(val) + elif isinstance(val, str): + return val.lower() in ('yes', 'true', 't', '1', 'y') + elif isinstance(val, int): + return val > 0 + else: + return False + + @staticmethod + def _coerce_datetime(val): + if isinstance(val, (datetime.date, datetime.datetime)): + return val + elif isinstance(val, int): + try: + return datetime.date(year=val, month=1, day=1) + except: + pass + return dateparser.parse(val) + + @property + def query_label(self) -> str: + if self._query_label: + return self._query_label + else: + return f'{self.predicate.replace("_", " ").title()}: {self.key.replace("_", " ").title()}' + + @property + def query_field(self) -> StringField: + self.render_kw.update({'data-entities': ",".join(Schema.__predicates_types__[self.predicate])}) + if self.type == bool: + return BooleanField(label=self.query_label, render_kw=self.render_kw) + elif self.type == int: + return IntegerField(label=self.query_label, render_kw=self.render_kw) + elif self.choices: + return TomSelectMutlitpleField(label=self.query_label, render_kw=self.render_kw, choices=self.choices) + else: + return StringField(label=self.query_label, render_kw=self.render_kw) + + + + + class _PrimitivePredicate: @@ -105,13 +249,14 @@ class _PrimitivePredicate: dgraph_predicate_type = 'string' is_list_predicate = False - comparison_operator = "eq" + default_operator = "eq" def __init__(self, label: str = None, default: str = None, required=False, overwrite=False, + facets=None, new=True, edit=True, queryable=False, @@ -124,7 +269,7 @@ def __init__(self, tom_select=False, render_kw: dict = None, predicate_alias: list = None, - comparison_operator: str = None) -> None: + comparison_operators: str = None) -> None: """ Contruct a new Primitive Predicate """ @@ -139,8 +284,22 @@ def __init__(self, self.query_label = query_label or label self.query_description = query_description self.permission = permission - if comparison_operator: - self.comparison_operator = comparison_operator + self.operators = comparison_operators + + # Facets: parameter should accept lists and single Facet objects + if isinstance(facets, Facet): + facets = [facets] + + if facets: + self.facets = {facet.key: facet for facet in facets} + else: + self.facets = None + + if isinstance(comparison_operators, dict): + comparison_operators = [(k, v) for k, v in comparison_operators.items()] + comparison_operators.insert(0, ('','')) + self.operators = comparison_operators + # WTF Forms self.required = required @@ -244,13 +403,13 @@ def query_filter(self, vals: Union[str, list], predicate=None, **kwargs) -> str: try: if self.is_list_predicate: - return " AND ".join([f'{self.comparison_operator}({predicate}, "{strip_query(val)}")' for val in vals]) + return " AND ".join([f'{self.default_operator}({predicate}, "{strip_query(val)}")' for val in vals]) else: vals_string = ", ".join([f'"{strip_query(val)}"' for val in vals]) if len(vals) == 1: - return f'{self.comparison_operator}({predicate}, {vals_string})' + return f'{self.default_operator}({predicate}, {vals_string})' else: - return f'{self.comparison_operator}({predicate}, [{vals_string}])' + return f'{self.default_operator}({predicate}, [{vals_string}])' except: return f'has({predicate})' @@ -442,7 +601,7 @@ class ReverseRelationship(_PrimitivePredicate): """ dgraph_predicate_type = 'uid' - comparison_operator = 'uid_in' + default_operator = 'uid_in' def __init__(self, predicate_name, @@ -465,6 +624,11 @@ def __init__(self, self.default_predicates = default_predicates + # Facets + if self.facets: + for facet in self.facets.values(): + facet.predicate = predicate_name + # WTForms # if we want the form field to show all choices automatically. self.autoload_choices = autoload_choices @@ -600,7 +764,7 @@ class MutualRelationship(_PrimitivePredicate): dgraph_predicate_type = 'uid' is_list_predicate = False - comparison_operator = "uid_in" + default_operator = "uid_in" def __init__(self, allow_new=False, @@ -743,7 +907,7 @@ class UIDPredicate(Predicate): dgraph_predicate_type = 'uid' is_list_predicate = False - comparison_operator = 'uid_in' + default_operator = 'uid_in' def __init__(self, *args, **kwargs) -> None: super().__init__(read_only=True, hidden=True, new=False, @@ -906,7 +1070,7 @@ class DateTime(Predicate): dgraph_predicate_type = 'datetime' is_list_predicate = False - comparison_operator = 'between' + default_operator = 'between' def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -950,7 +1114,7 @@ def query_filter(self, vals: Union[str, list, int], custom_operator: Union['lt', try: if isinstance(vals, list) and len(vals) > 1: vals = [self.validation_hook(val) for val in vals[:2]] - return f'{self.comparison_operator}({self.predicate}, "{vals[0].year}-01-01", "{vals[1].year}-12-31")' + return f'{self.default_operator}({self.predicate}, "{vals[0].year}-01-01", "{vals[1].year}-12-31")' else: if isinstance(vals, list): @@ -959,7 +1123,7 @@ def query_filter(self, vals: Union[str, list, int], custom_operator: Union['lt', if custom_operator: return f'{custom_operator}({self.predicate}, "{date.year}")' else: - return f'{self.comparison_operator}({self.predicate}, "{date.year}-01-01", "{date.year}-12-31")' + return f'{self.default_operator}({self.predicate}, "{date.year}-01-01", "{date.year}-12-31")' except: return f'has({self.predicate})' @@ -1078,7 +1242,7 @@ class SingleRelationship(Predicate): dgraph_predicate_type = 'uid' is_list_predicate = False - comparison_operator = 'uid_in' + default_operator = 'uid_in' def __init__(self, relationship_constraint=None, diff --git a/flaskinventory/flaskdgraph/query.py b/flaskinventory/flaskdgraph/query.py index dc62bc81..58b19181 100644 --- a/flaskinventory/flaskdgraph/query.py +++ b/flaskinventory/flaskdgraph/query.py @@ -22,7 +22,7 @@ def build_query_string(query: dict, public=True) -> str: Returns a dql query string with two queries: `total` and `q` """ - from flaskinventory.flaskdgraph.dgraph_types import MutualRelationship, SingleRelationship + from flaskinventory.flaskdgraph.dgraph_types import Facet, MutualRelationship, SingleRelationship # get parameter: maximum results per page try: @@ -77,31 +77,33 @@ def build_query_string(query: dict, public=True) -> str: # first we clean the query dict # make sure that the predicates exists (cannot query arbitrary predicates) and is queryable # also asserts that certain predicates remain private (e.g., email addresses) - cleaned_query = {k: v for k, v in query.items( + _cleaned_query = {k: v for k, v in query.items( ) if k in Schema.get_queryable_predicates()} + cleaned_query = {Schema.get_queryable_predicates()[k]: v for k, v in _cleaned_query.items() if not isinstance(Schema.get_queryable_predicates()[k], Facet)} + + # preprare facets + facets = {Schema.get_queryable_predicates()[k]: v for k, v in _cleaned_query.items() if isinstance(Schema.get_queryable_predicates()[k], Facet)} + for facet in facets: + if facet.predicate not in _cleaned_query: + cleaned_query.update({Schema.predicates()[facet.predicate]: None}) + operators = {k.split('*')[0]: v[0] for k, v in query.items() if '*operator' in k} - facets = {k.split('|')[0]: {k.split('|')[1]: v[0]} - for k, v in query.items() if '|' in k and not '*' in k} - - for k in facets.keys(): - if k not in cleaned_query.keys() and k in Schema.get_queryable_predicates(): - cleaned_query.update({k: None}) # prevent querying everything if len(cleaned_query) == 0 and len(filters) == 0: return False query_parts = ['uid', 'unique_name', 'name', 'dgraph.type', 'authors', 'other_names', 'published_date'] + query_parts_total = ['count(uid)'] if public: filters.append('eq(entry_review_status, "accepted")') - for key, val in cleaned_query.items(): + for predicate, val in cleaned_query.items(): # get predicate from Schema - predicate = Schema.get_queryable_predicates()[key] # check if we have a non-default operator - operator = operators.get(key, None) + operator = operators.get(predicate.predicate, None) # Let the predicate object generate the filter query part predicate_filter = predicate.query_filter(val, custom_operator=operator) @@ -121,25 +123,28 @@ def build_query_string(query: dict, public=True) -> str: # check if we have facet filters # "predicate|facet*operator": "name of operator" # "audience_size|subscribers*operator": "gt" - if key in facets.keys(): - for facet, subvalue in facets[key].items(): - facet_operator = operators.get(f'{key}|{facet}', 'eq') - facet_filter.append(f'{facet_operator}({facet}, {subvalue})') - facet_list.append(facet) + for facet, facet_value in facets.items(): + if facet.predicate == predicate.predicate: + facet_operator = operators.get(f'{facet}', 'eq') + filt = facet.query_filter(facet_value, operator=facet_operator) + if filt: + facet_filter.append(filt) + facet_list.append(facet.key) if len(facet_filter) > 0: facet_filter = f'@facets({" AND ".join(facet_filter)})' facet_list = f'@facets({", ".join(facet_list)})' + query_parts_total.append(f'{predicate.query} {facet_filter}') else: facet_filter = '' facet_list = '' if isinstance(predicate, (SingleRelationship, MutualRelationship)): query_parts.append( - f'{predicate.query} {facet_filter} {facet_list} {{ uid name unique_name }}') + f'{predicate.query} {facet_filter} {facet_list} {{ uid name unique_name }}'.strip()) else: query_parts.append( - f'{predicate.query} {facet_filter} {facet_list}') + f'{predicate.query} {facet_filter} {facet_list}'.strip()) filters = " AND ".join(filters) @@ -151,7 +156,8 @@ def build_query_string(query: dict, public=True) -> str: query_parts = list(set(query_parts)) if len(facets.keys()) > 0: - cascade = "@cascade" + cascade = list(set([facet.predicate for facet in facets])) + cascade = f"@cascade({', '.join(cascade)})" else: cascade = "" @@ -166,10 +172,10 @@ def build_query_string(query: dict, public=True) -> str: {{ total(func: has(dgraph.type)) @filter({filters}) {cascade} {{ - count(uid) + {" ".join(query_parts_total)} }} - q(func: has(dgraph.type), orderasc: unique_name, first: {max_results}, offset: {page * max_results}) + q(func: has(dgraph.type), orderasc: name, first: {max_results}, offset: {page * max_results}) @filter({filters}) {cascade} {{ {" ".join(query_parts)} }} @@ -205,20 +211,12 @@ def get_field(self, field): for k, v in fields.items(): if not hasattr(F, k): setattr(F, k, v.query_field) + if isinstance(v.operators, list): + operator_selection = SelectField('operator', name=f'{v}*operator', choices=v.operators) + setattr(F, f'{k}*operator', operator_selection) - # add pagination parameters: max_results - if '_max_results' in populate_obj: - if isinstance(populate_obj['_max_results'], list): - populate_obj['max_results'] = populate_obj['_max_results'][0] - else: - populate_obj['max_results'] = populate_obj['_max_results'] - if '_terms' in populate_obj: - if isinstance(populate_obj['_terms'], list): - populate_obj['terms'] = populate_obj['_terms'][0] - else: - populate_obj['terms'] = populate_obj['_terms'] - form = F(data=populate_obj) + form = F(formdata=populate_obj) return form diff --git a/flaskinventory/flaskdgraph/schema.py b/flaskinventory/flaskdgraph/schema.py index 449614bb..b2382d7a 100644 --- a/flaskinventory/flaskdgraph/schema.py +++ b/flaskinventory/flaskdgraph/schema.py @@ -44,17 +44,18 @@ class Schema: def __init_subclass__(cls) -> None: - from .dgraph_types import Predicate, SingleRelationship, ReverseRelationship, MutualRelationship + from .dgraph_types import _PrimitivePredicate, Facet, Predicate, SingleRelationship, ReverseRelationship, MutualRelationship predicates = {key: getattr(cls, key) for key in cls.__dict__ if isinstance(getattr(cls, key), (Predicate, MutualRelationship))} relationship_predicates = {key: getattr(cls, key) for key in cls.__dict__ if isinstance(getattr(cls, key), (SingleRelationship, MutualRelationship))} reverse_predicates = {key: getattr(cls, key) for key in cls.__dict__ if isinstance(getattr(cls, key), ReverseRelationship)} + # base list of queryable predicates queryable_predicates = {key: val for key, val in predicates.items() if val.queryable} + # add reverse predicates that can be queried queryable_predicates.update({val._predicate: val for key, val in reverse_predicates.items() if val.queryable}) - Schema.__queryable_predicates__.update(queryable_predicates) # inherit predicates from parent classes for parent in cls.__bases__: @@ -84,14 +85,16 @@ def __init_subclass__(cls) -> None: Schema.__perm_registry_new__[cls.__name__] = cls.__permission_new__ Schema.__perm_registry_edit__[cls.__name__] = cls.__permission_edit__ - Schema.__queryable_predicates_by_type__[cls.__name__] = {key: val for key, val in predicates.items() if val.queryable} - Schema.__queryable_predicates_by_type__[cls.__name__].update({val._predicate: val for key, val in reverse_predicates.items() if val.queryable}) - # Bind and "activate" predicates for initialized class for key in cls.__dict__: attribute = getattr(cls, key) if isinstance(attribute, (Predicate, MutualRelationship)): setattr(attribute, 'predicate', key) + if attribute.facets: + for facet in attribute.facets.values(): + facet.predicate = key + if facet.queryable: + queryable_predicates.update({str(facet): facet}) if key not in cls.__predicates_types__: cls.__predicates_types__.update({key: [cls.__name__]}) else: @@ -112,6 +115,13 @@ def __init_subclass__(cls) -> None: if cls.__name__ not in cls.__reverse_predicates_types__[attribute.predicate]: cls.__reverse_predicates_types__[attribute.predicate].append(cls.__name__) + Schema.__queryable_predicates__.update(queryable_predicates) + + Schema.__queryable_predicates_by_type__[cls.__name__] = {key: val for key, val in predicates.items() if val.queryable} + Schema.__queryable_predicates_by_type__[cls.__name__].update({key: val for key, val in queryable_predicates.items() if isinstance(val, Facet)}) + Schema.__queryable_predicates_by_type__[cls.__name__].update({val._predicate: val for key, val in reverse_predicates.items() if val.queryable}) + + @classmethod def get_types(cls) -> list: """ diff --git a/flaskinventory/main/model.py b/flaskinventory/main/model.py index fd33898e..96dae9e0 100644 --- a/flaskinventory/main/model.py +++ b/flaskinventory/main/model.py @@ -3,14 +3,14 @@ from flask import current_app from flaskinventory import dgraph from flaskinventory.errors import InventoryPermissionError, InventoryValidationError -from flaskinventory.flaskdgraph.dgraph_types import (MutualListRelationship, String, Integer, Boolean, UIDPredicate, +from flaskinventory.flaskdgraph.dgraph_types import (String, Integer, Boolean, UIDPredicate, SingleChoice, MultipleChoice, DateTime, Year, GeoScalar, - ListString, ListRelationship, - Geo, SingleRelationship, UniqueName, + ListString, + SingleRelationship, ListRelationship, MutualListRelationship, + Geo, UniqueName, ReverseRelationship, ReverseListRelationship, - NewID, UID, Scalar) - + NewID, UID, Scalar, Facet) from flaskinventory.add.external import geocode, reverse_geocode, get_wikidata from flaskinventory.users.constants import USER_ROLES @@ -380,7 +380,7 @@ class Entry(Schema): overwrite=True, new=False) - description = String(large_textfield=True, queryable=True, comparison_operator="alloftext") + description = String(large_textfield=True) entry_notes = String(description='Do you have any other notes on the entry that you just coded?', large_textfield=True) @@ -604,7 +604,12 @@ class Source(Entry): radio_field=True, queryable=True) - audience_size = Year(default=datetime.date.today(), edit=False, queryable=True) + audience_size = Year(default=datetime.date.today(), + edit=False, + facets=[Facet("unit", queryable=True, choices=['followers', 'subscribers', 'copies sold']), + Facet("count", dtype=int, queryable=True, comparison_operators={'gt': 'greater', 'lt': 'less'}), + Facet("data_from")] + ) publishes_org = OrganizationAutocode('publishes', label='Published by', @@ -870,7 +875,8 @@ class Tool(Entry): published_date = Year(label='Year of publication', description="Which year was the tool published?", - queryable=True) + queryable=True, + comparison_operators={'ge': 'after', 'le': 'before', 'eq': 'exact'}) last_updated = DateTime(description="When was the tool last updated?", new=False) diff --git a/flaskinventory/templates/helpers/_formhelpers.html b/flaskinventory/templates/helpers/_formhelpers.html index 512c267a..c2821fed 100644 --- a/flaskinventory/templates/helpers/_formhelpers.html +++ b/flaskinventory/templates/helpers/_formhelpers.html @@ -72,7 +72,7 @@ {% macro render_query_field(field) %} {% if field %} - {% if field.type not in ['SubmitField', 'CSRFTokenField'] and not field.name.startswith('_') %} + {% if field.type not in ['SubmitField', 'CSRFTokenField'] and not field.name.startswith('_') and not '*operator' in field.name %} {% if field.type == 'BooleanField' %} {% if field.description %}

{{ field.description }}

diff --git a/flaskinventory/templates/query/sidebar.html b/flaskinventory/templates/query/sidebar.html index 8b26ea6b..cf524145 100644 --- a/flaskinventory/templates/query/sidebar.html +++ b/flaskinventory/templates/query/sidebar.html @@ -15,9 +15,69 @@
Query
@@ -45,7 +105,7 @@
Query
let all = document.querySelectorAll('[data-entities]') for (element of all) { if (!alwaysVisible.includes(element.id)) { - element.parentElement.hidden = true + document.getElementById(`${element.id}-container`).hidden = true } } } @@ -61,7 +121,7 @@
Query
if (o.selected) { let fields = document.querySelectorAll(`[data-entities*="${o.value}"]`) for (f of fields) { - f.parentElement.hidden = false + document.getElementById(`${f.id}-container`).hidden = false } } } @@ -105,20 +165,20 @@
Query
function rearrange() { for (predicate of correctOrder) { - let el = document.getElementById(predicate).parentElement + let el = document.getElementById(`${predicate}-container`) sortedContainer.appendChild(el) } let all = document.querySelectorAll('[data-entities]') for (el of all) { + container = document.getElementById(`${el.id}-container`) if (el.checked) { - if (el.parentElement.parentElement.id != sortedContainer.id) { - sortedContainer.appendChild(el.parentElement) + if (container != sortedContainer.id) { + sortedContainer.appendChild(container) } } if (el.value && el.type !== 'checkbox') { - console.log(el) - if (el.parentElement.parentElement.id != sortedContainer.id) { - sortedContainer.appendChild(el.parentElement) + if (container != sortedContainer.id) { + sortedContainer.appendChild(container) } } } diff --git a/flaskinventory/templates/query/tomselect.html b/flaskinventory/templates/query/tomselect.html index 78741e74..e2275d91 100644 --- a/flaskinventory/templates/query/tomselect.html +++ b/flaskinventory/templates/query/tomselect.html @@ -6,7 +6,7 @@ {% if field.type == "TomSelectMutlitpleField" %} - new TomSelect('#{{ field.id.replace('.', '\\\.').replace('~', '\\\~') }}', + new TomSelect('#{{ field.id.replace('.', '\\\.').replace('~', '\\\~').replace('|', '\\\|') }}', { create: false, plugins: ['remove_button'], diff --git a/flaskinventory/view/routes.py b/flaskinventory/view/routes.py index d548c17d..b826a4c2 100644 --- a/flaskinventory/view/routes.py +++ b/flaskinventory/view/routes.py @@ -152,13 +152,13 @@ def query(): r_args = {k: v for k, v in request.args.to_dict(flat=False).items() if v[0] != ''} - if '_page' in r_args: + try: current_page = int(r_args.pop('_page')[0]) - else: + except: current_page = 1 form = generate_query_forms(dgraph_types=['Source', 'Organization', 'Tool', 'Archive', 'Dataset', 'Corpus'], - populate_obj=r_args) + populate_obj=request.args) return render_template("query/index.html", form=form, result=result, r_args=r_args, total=total, pages=pages, current_page=current_page) diff --git a/tests/test_queries.py b/tests/test_queries.py index 99229685..752eb7d4 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -268,7 +268,10 @@ def test_facet_filters(self): print('-- test_facet_filters() --\n') with self.client as c: query_string = {"audience_size|papers_sold": 52000, - "audience_size|papers_sold*operator": 'gt'} + "audience_size|papers_sold*operator": 'gt', + "audience_size|unit": "papers sold", + "audience_size|count": 52000, + "audience_size|count*operator": 'gt'} response = c.get(f'/query/development/json', query_string=query_string) self.assertEqual(response.json[0]['unique_name'], 'derstandard_print')