Skip to content

Commit

Permalink
Search improvements: souped-up explore page (#412)
Browse files Browse the repository at this point in the history 
Can edit boost vars and view subquery boosts in action from the page
  • Loading branch information
@marcelkornblum
marcelkornblum authored Jul 28, 2023
1 parent 1c696de commit d7919d0
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 32 deletions.
8 changes: 4 additions & 4 deletions src/extended_search/managers/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,17 @@ def get_indexed_field_name(model_field_name, analyzer):
return f"{model_field_name}{field_name_suffix}"


def get_search_query(cls, query_str, model_class, *args, **kwargs):
def get_search_query(index_manager, query_str, model_class, *args, **kwargs):
"""
Uses the field mapping to derive the full nested SearchQuery
"""
query = None
for field_mapping in cls.get_mapping():
query_elements = cls._get_search_query_from_mapping(
for field_mapping in index_manager.get_mapping():
query_elements = index_manager._get_search_query_from_mapping(
query_str, model_class, field_mapping
)
if query_elements is not None:
query = cls._add_to_query(
query = index_manager._add_to_query(
query,
query_elements,
)
Expand Down
2 changes: 1 addition & 1 deletion src/extended_search/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ def __init__(self, *args, **kwargs):
def _get_all_indexed_fields(self):
fields = {}
for model_cls in get_indexed_models():
for search_field in model_cls.search_fields:
for search_field in model_cls.get_searchable_search_fields():
definition_cls = search_field.get_definition_model(model_cls)
if definition_cls not in fields:
fields[definition_cls] = set()
Expand Down
4 changes: 2 additions & 2 deletions src/extended_search/tests/test_settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,12 @@ def test_get_all_indexed_fields(self, mocker):
mock_searchfield_2.get_definition_model.return_value = "--model--"
mock_searchfield_3 = mocker.MagicMock()
mock_searchfield_3.get_definition_model.return_value = "--second-model--"
mock_model_1.search_fields = [
mock_model_1.get_searchable_search_fields.return_value = [
mock_searchfield_1,
mock_searchfield_2,
mock_searchfield_2,
]
mock_model_2.search_fields = [
mock_model_2.get_searchable_search_fields.return_value = [
mock_searchfield_3,
]
mock_get_models.return_value = [
Expand Down
148 changes: 136 additions & 12 deletions src/search/templates/search/explore.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,47 @@
{% endblock location %}

{% block content %}
{% if perms.search.view_explore %}
{% if messages %}
{% for message in messages %}
{% if message.tags == 'info' %}
<div class="govuk-notification-banner"
role="region"
aria-labelledby="govuk-notification-banner-title"
data-module="govuk-notification-banner">
<div class="govuk-notification-banner__header">
<h2 class="govuk-notification-banner__title"
id="govuk-notification-banner-title">Success</h2>
</div>
<div class="govuk-notification-banner__content">
<p class="govuk-notification-banner__heading">{{ message }}</p>
</div>
</div>
{% elif message.tags == 'error' %}
<div class="govuk-error-summary" data-module="govuk-error-summary">
<div role="alert">
<h2 class="govuk-error-summary__title">There is a problem</h2>
<div class="govuk-error-summary__body">
<ul class="govuk-list govuk-error-summary__list">
<li>
<a href="#">{{ message }}</a>
</li>
</ul>
</div>
</div>
</div>
{% endif %}
{% endfor %}
{% endif %}
{% if perms.extended_search.view_explore %}
<div class="govuk-grid-column-full">
<div class="govuk-warning-text">
<span class="govuk-warning-text__icon" aria-hidden="true">!</span>
<strong class="govuk-warning-text__text">
<span class="govuk-warning-text__assistive">Warning</span>
This is a complex area. Read up on <a href="https://www.elastic.co/blog/how-to-improve-elasticsearch-search-relevance-with-boolean-queries">the theory</a> behind, and <a href="https://github.blog/2023-03-09-how-github-docs-new-search-works/">the basis of</a>, <a href="https://github.com/uktrade/digital-workspace-v2/blob/main/docs/search.md">our approach</a>.
</strong>
</div>

<h1 class="govuk-heading-m">Configure</h1>
<div class="govuk-accordion"
data-module="govuk-accordion"
Expand All @@ -29,9 +68,33 @@ <h3 class="govuk-heading-s">Boost variables</h3>
<div id="accordion-default-content-1"
class="govuk-accordion__section-content"
aria-labelledby="accordion-default-heading-1">
<ul>
{% for boost in boost_variables %}<li>{{ boost.name }}: {{ boost.value }}</li>{% endfor %}
</ul>
<div class="govuk-summary-card">
<div class="govuk-summary-card__content">
<dl class="govuk-summary-list" style="font-size: 1rem; line-height: 1">
{% for boost in boost_variables %}
<div class="govuk-summary-list__row">
<dt class="govuk-summary-list__key" style="width:75%; font-weight:400">{{ boost.name }}</dt>
<dd class="govuk-summary-list__value">
{% if perms.extended_search.change_setting %}
<form method="post">
{% csrf_token %}
<input type="hidden" name="key" value="{{ boost.name }}">
<input type="text"
name="value"
value="{{ boost.value }}"
class="govuk-input"
style="width:4em">
<input type="submit" value="Save" class="govuk-button" style="margin:0">
</form>
{% else %}
{{ boost.value }}
{% endif %}
</dd>
</div>
{% endfor %}
</dl>
</div>
</div>
</div>
</div>
<div class="govuk-accordion__section">
Expand All @@ -46,12 +109,73 @@ <h3 class="govuk-heading-s">Subqueries</h3>
<div id="accordion-default-content-1"
class="govuk-accordion__section-content"
aria-labelledby="accordion-default-heading-2">
{% for type in sub_queries %}
<h2 class="govuk-heading-s">{{ type.name|capfirst }}</h2>
<ul>
{% for query in type.queries %}<li>{{ query.id }}: {{ query.value }}</li>{% endfor %}
</ul>
{% endfor %}
<h2 class="govuk-heading-s">All pages</h2>

<table class="govuk-table" style="font-size:1rem">
<thead class="govuk-table__head">
<tr class="govuk-table__row">
<th scope="col" class="govuk-table__header">Field</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Analyzer</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Query type</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Combined boost</th>
</tr>
</thead>
<tbody class="govuk-table__body">
{% for query in sub_queries.pages %}
<tr class="govuk-table__row">
<th scope="row" class="govuk-table__header">{{ query.field }}</th>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.analyzer }}</td>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.query_type }}</td>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.boost|floatformat:1 }}</td>
</tr>
{% endfor %}
</tbody>
</table>

<h2 class="govuk-heading-s">People</h2>

<table class="govuk-table" style="font-size:1rem">
<thead class="govuk-table__head">
<tr class="govuk-table__row">
<th scope="col" class="govuk-table__header">Field</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Analyzer</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Query type</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Combined boost</th>
</tr>
</thead>
<tbody class="govuk-table__body">
{% for query in sub_queries.people %}
<tr class="govuk-table__row">
<th scope="row" class="govuk-table__header">{{ query.field }}</th>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.analyzer }}</td>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.query_type }}</td>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.boost|floatformat:1 }}</td>
</tr>
{% endfor %}
</tbody>
</table>
<h2 class="govuk-heading-s">Teams</h2>

<table class="govuk-table" style="font-size:1rem">
<thead class="govuk-table__head">
<tr class="govuk-table__row">
<th scope="col" class="govuk-table__header">Field</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Analyzer</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Query type</th>
<th scope="col" class="govuk-table__header govuk-table__header--numeric">Combined boost</th>
</tr>
</thead>
<tbody class="govuk-table__body">
{% for query in sub_queries.teams %}
<tr class="govuk-table__row">
<th scope="row" class="govuk-table__header">{{ query.field }}</th>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.analyzer }}</td>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.query_type }}</td>
<td class="govuk-table__cell govuk-table__cell--numeric">{{ query.boost|floatformat:1 }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
Expand All @@ -63,8 +187,8 @@ <h1 class="govuk-heading-m">Evaluate</h1>
data-module="govuk-accordion"
id="accordion-default">
{% search_category category='all_pages' show_heading=True %}
{% comment %} {% search_category category='people' show_heading=True %}
{% search_category category='teams' show_heading=True %} {% endcomment %}
{% search_category category='people' show_heading=True %}
{% search_category category='teams' show_heading=True %}
</div>
</div>
{% else %}
Expand Down
10 changes: 3 additions & 7 deletions src/search/templates/search/partials/search_results_single_category.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,8 @@

{% if search_category in "guidance,tools,news" %}
<div class="search-results__all">
<div class="left-col">
{% endif %}

{% search_category category=search_category %}

{% if search_category in "guidance,tools,news" %}
</div>
<div class="left-col">{% search_category category=search_category %}</div>
</div>
{% else %}
{% search_category category=search_category %}
{% endif %}
87 changes: 81 additions & 6 deletions src/search/views.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,30 @@
import logging

from django.contrib.auth.decorators import user_passes_test
from django.contrib import messages
from django.http import HttpRequest, HttpResponse
from django.shortcuts import redirect
from django.template.response import TemplateResponse
from django.urls import reverse
from django.views.decorators.http import require_http_methods

from search.templatetags.search import SEARCH_CATEGORIES
from wagtail.search.query import Or, Phrase, PlainText, Fuzzy

from content.models import ContentPage, ContentPageIndexManager
from peoplefinder.models import Person, PersonIndexManager, Team, TeamIndexManager
from extended_search.backends.query import OnlyFields
from extended_search.models import Setting as SearchSetting
from extended_search.settings import extended_search_settings
from search.templatetags.search import SEARCH_CATEGORIES


logger = logging.getLogger(__name__)


def can_view_explore():
return user_passes_test(lambda u: u.has_perm("extended_search.view_explore"))


@require_http_methods(["GET"])
def search(request: HttpRequest, category: str = None) -> HttpResponse:
query = request.GET.get("query", "")
Expand All @@ -33,10 +46,21 @@ def search(request: HttpRequest, category: str = None) -> HttpResponse:
return TemplateResponse(request, "search/search.html", context=context)


@can_view_explore()
def explore(request: HttpRequest) -> HttpResponse:
"""
Administrative view for exploring search options, boosts, etc
"""
if request.method == "POST":
if not request.user.has_perm("extended_search.change_setting"):
messages.error(request, "You are not authorised to edit settings")

key = request.POST.get("key")
value = request.POST.get("value")

SearchSetting.objects.update_or_create(key=key, defaults={"value": value})
messages.info(request, f"Setting '{key}' saved")

query = request.GET.get("query", "")
page = request.GET.get("page", "1")

Expand All @@ -46,17 +70,68 @@ def explore(request: HttpRequest) -> HttpResponse:
if "boost_parts" in k
]

subqueries = {"pages": [], "people": [], "teams": []}
analyzer_field_suffices = [
(k, v["index_fieldname_suffix"])
for k, v in extended_search_settings["analyzers"].items()
]
for mapping in ContentPageIndexManager.get_mapping():
field = ContentPageIndexManager._get_search_query_from_mapping(
query, ContentPage, mapping
)
get_query_info(subqueries["pages"], field, mapping, analyzer_field_suffices)
for mapping in PersonIndexManager.get_mapping():
field = PersonIndexManager._get_search_query_from_mapping(
query, Person, mapping
)
get_query_info(subqueries["people"], field, mapping, analyzer_field_suffices)
for mapping in TeamIndexManager.get_mapping():
field = TeamIndexManager._get_search_query_from_mapping(query, Team, mapping)
get_query_info(subqueries["teams"], field, mapping, analyzer_field_suffices)

context = {
"search_url": reverse("search:explore"),
"search_query": query,
"search_category": "all",
"page": page,
"boost_variables": boost_vars,
"sub_queries": [
{"name": "pages", "queries": [{"id": 1, "value": "{match_all: {}}"}]},
{"name": "people", "queries": [{"id": 1, "value": "{match_all: {}}"}]},
{"name": "teams", "queries": [{"id": 1, "value": "{match_all: {}}"}]},
],
"sub_queries": subqueries,
}

return TemplateResponse(request, "search/explore.html", context=context)


def get_query_info(fields, field, mapping, suffix_map):
if field is None:
return fields

if isinstance(field, Or):
for f in field.subqueries:
fields = get_query_info(fields, f, mapping, suffix_map)

elif isinstance(field, OnlyFields):
core_field = field.subquery.subquery

analyzer_name = "tokenizer"
for analyzer, suffix in suffix_map:
if suffix and suffix in field.fields[0]:
analyzer_name = analyzer

if isinstance(core_field, Phrase):
query_type = "phrase"
elif isinstance(core_field, Fuzzy):
query_type = "fuzzy"
elif isinstance(core_field, PlainText):
if core_field.operator == "and":
query_type = "query_and"
else:
query_type = "query_or"
fields.append(
{
"query_type": query_type,
"field": mapping["model_field_name"],
"analyzer": analyzer_name,
"boost": field.subquery.boost,
}
)
return fields

0 comments on commit d7919d0

Please sign in to comment.