Skip to content

Commit

Permalink
Merge branch 'master' into topic-landing-pools
Browse files Browse the repository at this point in the history
  • Loading branch information
nsantacruz committed Dec 2, 2024
2 parents 49aa8c7 + 1857fc4 commit affb67e
Show file tree
Hide file tree
Showing 37 changed files with 399 additions and 214 deletions.
16 changes: 16 additions & 0 deletions api/api_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""
Classes for API errors
"""
from sefaria.client.util import jsonResponse


class APIInvalidInputException(Exception):
"""
When data in an invalid format is passed to an API
"""
def __init__(self, message):
super().__init__(message)
self.message = message

def to_json_response(self):
return jsonResponse({"invalid_input_error": self.message}, status=400)
11 changes: 9 additions & 2 deletions api/views.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from sefaria.model import *
from sefaria.model.text_reuqest_adapter import TextRequestAdapter
from sefaria.client.util import jsonResponse
from sefaria.system.exceptions import InputError, ComplexBookLevelRefError
from django.views import View
from .api_warnings import *

Expand Down Expand Up @@ -53,6 +54,12 @@ def get(self, request, *args, **kwargs):
if return_format not in self.RETURN_FORMATS:
return jsonResponse({'error': f'return_format should be one of those formats: {self.RETURN_FORMATS}.'}, status=400)
text_manager = TextRequestAdapter(self.oref, versions_params, fill_in_missing_segments, return_format)
data = text_manager.get_versions_for_query()
data = self._handle_warnings(data)

try:
data = text_manager.get_versions_for_query()
data = self._handle_warnings(data)

except Exception as e:
return jsonResponse({'error': str(e)}, status=400)

return jsonResponse(data)
18 changes: 18 additions & 0 deletions docs/openAPI.json
Original file line number Diff line number Diff line change
Expand Up @@ -4460,6 +4460,24 @@
"tags": [
"Topic"
],
"parameters": [
{
"examples": {
"Return all topics": {
"value": "limit=0"
},
"Return 20 topics": {
"value": "limit=20"
}
},
"name": "limit",
"description": "This parameter limits the number of topics returned. The default is `1000`. If `limit=0` then all topics will be returned.",
"schema": {
"type": "integer"
},
"in": "query"
}
],
"responses": {
"200": {
"content": {
Expand Down
5 changes: 5 additions & 0 deletions helm-chart/sefaria-project/templates/rollout/task.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ spec:
value: "varnish-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
- name: HELM_REVISION
value: "{{ .Release.Revision }}"
- name: SLACK_URL
valueFrom:
secretKeyRef:
name: { { template "sefaria.secrets.slackWebhook" . } }
key: slack-webhook
envFrom:
{{- if .Values.tasks.enabled }}
- secretRef:
Expand Down
4 changes: 2 additions & 2 deletions helm-chart/sefaria-project/templates/rollout/web.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,12 @@ spec:
fieldPath: spec.nodeName
- name: OTEL_RESOURCE_ATTRIBUTES
value: k8s.container.name=app,k8s.deployment.name={{ .Values.deployEnv }}-web,k8s.namespace.name={{ .Release.Namespace }},k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)
{{- end }}
- name: SLACK_URL
valueFrom:
secretKeyRef:
name: { { template "sefaria.secrets.slackWebhook" . } }
name: {{ template "sefaria.secrets.slackWebhook" . }}
key: slack-webhook
{{- end }}
envFrom:
{{- if .Values.tasks.enabled }}
- secretRef:
Expand Down
9 changes: 6 additions & 3 deletions reader/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
from sefaria.site.site_settings import SITE_SETTINGS
from sefaria.system.multiserver.coordinator import server_coordinator
from sefaria.system.decorators import catch_error_as_json, sanitize_get_params, json_response_decorator
from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError
from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError, ComplexBookLevelRefError
from sefaria.system.cache import django_cache
from sefaria.system.database import db
from sefaria.helper.search import get_query_obj
Expand Down Expand Up @@ -1418,8 +1418,11 @@ def _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=comment
return text

if not multiple or abs(multiple) == 1:
text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad,
alts=alts, wrapLinks=wrapLinks, layer_name=layer_name)
try:
text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad,
alts=alts, wrapLinks=wrapLinks, layer_name=layer_name)
except Exception as e:
return jsonResponse({'error': str(e)}, status=400)
return jsonResponse(text, cb)
else:
# Return list of many sections
Expand Down
50 changes: 45 additions & 5 deletions sefaria/helper/linker.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,56 @@
import json
import spacy
import structlog
from cerberus import Validator
from sefaria.model.linker.ref_part import TermContext, RefPartType
from sefaria.model.linker.ref_resolver import PossiblyAmbigResolvedRef
from sefaria.model import text, library
from sefaria.model.webpage import WebPage
from sefaria.system.cache import django_cache
from typing import List, Union, Optional, Tuple
from api.api_errors import APIInvalidInputException
from typing import List, Optional, Tuple

logger = structlog.get_logger(__name__)

FIND_REFS_POST_SCHEMA = {
"text": {
"type": "dict",
"required": True,
"schema": {
"title": {"type": "string", "required": True},
"body": {"type": "string", "required": True},
},
},
"metaDataForTracking": {
"type": "dict",
"required": False,
"schema": {
"url": {"type": "string", "required": False},
"description": {"type": "string", "required": False},
"title": {"type": "string", "required": False},
},
},
"lang": {
"type": "string",
"allowed": ["he", "en"],
"required": False,
},
"version_preferences_by_corpus": {
"type": "dict",
"required": False,
"nullable": True,
"keysrules": {"type": "string"},
"valuesrules": {
"type": "dict",
"schema": {
"type": "string",
"keysrules": {"type": "string"},
"valuesrules": {"type": "string"},
},
},
},
}


def load_spacy_model(path: str) -> spacy.Language:
import re, tarfile
Expand Down Expand Up @@ -64,13 +105,12 @@ class _FindRefsText:
body: str
lang: str

# def __post_init__(self):
# from sefaria.utils.hebrew import is_mostly_hebrew
# self.lang = 'he' if is_mostly_hebrew(self.body) else 'en'


def _unpack_find_refs_request(request):
validator = Validator(FIND_REFS_POST_SCHEMA)
post_body = json.loads(request.body)
if not validator.validate(post_body):
raise APIInvalidInputException(validator.errors)
meta_data = post_body.get('metaDataForTracking')
return _create_find_refs_text(post_body), _create_find_refs_options(request.GET, post_body), meta_data

Expand Down
23 changes: 21 additions & 2 deletions sefaria/helper/tests/linker_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from sefaria.model.text import Ref, TextChunk
from sefaria.model.webpage import WebPage
from sefaria.settings import ENABLE_LINKER
from api.api_errors import APIInvalidInputException

if not ENABLE_LINKER:
pytest.skip("Linker not enabled", allow_module_level=True)
Expand Down Expand Up @@ -80,6 +81,12 @@ def mock_request_post_data_without_meta_data(mock_request_post_data: dict) -> di
return mock_request_post_data


@pytest.fixture
def mock_request_invalid_post_data(mock_request_post_data: dict) -> dict:
mock_request_post_data['text'] = 'plain text'
return mock_request_post_data


def make_mock_request(post_data: dict) -> WSGIRequest:
factory = RequestFactory()
request = factory.post('/api/find-refs', data=json.dumps(post_data), content_type='application/json')
Expand Down Expand Up @@ -109,6 +116,11 @@ def mock_request_without_meta_data(mock_request_post_data_without_meta_data: dic
return make_mock_request(mock_request_post_data_without_meta_data)


@pytest.fixture
def mock_request_invalid(mock_request_invalid_post_data: dict) -> WSGIRequest:
return make_mock_request(mock_request_invalid_post_data)


@pytest.fixture
def mock_webpage() -> WebPage:
# Note, the path of WebPage matches the path of the import we want to patch
Expand Down Expand Up @@ -162,6 +174,13 @@ def test_make_find_refs_response_without_meta_data(self, mock_request_without_me
mock_webpage.add_hit.assert_not_called()
mock_webpage.save.assert_not_called()

def test_make_find_refs_response_invalid_post_data(self, mock_request_invalid: dict,
mock_webpage: Mock):
with pytest.raises(APIInvalidInputException) as exc_info:
response = linker.make_find_refs_response(mock_request_invalid)
# assert that the 'text' field had a validation error
assert 'text' in exc_info.value.args[0]


class TestUnpackFindRefsRequest:
def test_unpack_find_refs_request(self, mock_request: WSGIRequest):
Expand Down Expand Up @@ -198,8 +217,8 @@ def mock_get_linker(self, spacy_model: spacy.Language):
with patch.object(library, 'get_linker') as mock_get_linker:
mock_linker = Mock()
mock_get_linker.return_value = mock_linker
mock_linker.link.return_value = LinkedDoc('', [], [])
mock_linker.link_by_paragraph.return_value = LinkedDoc('', [], [])
mock_linker.link.return_value = LinkedDoc('', [], [], [])
mock_linker.link_by_paragraph.return_value = LinkedDoc('', [], [], [])
yield mock_get_linker

def test_make_find_refs_response_linker_v3(self, mock_get_linker: WSGIRequest,
Expand Down
10 changes: 6 additions & 4 deletions sefaria/model/text.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import sefaria.system.cache as scache
from sefaria.system.cache import in_memory_cache
from sefaria.system.exceptions import InputError, BookNameError, PartialRefInputError, IndexSchemaError, \
NoVersionFoundError, DictionaryEntryNotFoundError, MissingKeyError
NoVersionFoundError, DictionaryEntryNotFoundError, MissingKeyError, ComplexBookLevelRefError
from sefaria.utils.hebrew import has_hebrew, is_all_hebrew, hebrew_term
from sefaria.utils.util import list_depth, truncate_string
from sefaria.datatype.jagged_array import JaggedTextArray, JaggedArray
Expand Down Expand Up @@ -1700,7 +1700,7 @@ def __init__(self, oref, lang, vtitle, merge_versions=False, versions=None):
elif oref.has_default_child(): #use default child:
self.oref = oref.default_child_ref()
else:
raise InputError("Can not get TextRange at this level, please provide a more precise reference")
raise ComplexBookLevelRefError(book_ref=oref.normal())
self.lang = lang
self.vtitle = vtitle
self.merge_versions = merge_versions
Expand Down Expand Up @@ -2434,7 +2434,7 @@ def __init__(self, oref, context=1, commentary=True, version=None, lang=None,
self._alts = []

if not isinstance(oref.index_node, JaggedArrayNode) and not oref.index_node.is_virtual:
raise InputError("Can not get TextFamily at this level, please provide a more precise reference")
raise InputError("Unable to find text for that ref")

for i in range(0, context):
oref = oref.context_ref()
Expand Down Expand Up @@ -4000,8 +4000,10 @@ def padded_ref(self):
except AttributeError: # This is a schema node, try to get a default child
if self.has_default_child():
return self.default_child_ref().padded_ref()
elif self.is_book_level():
raise ComplexBookLevelRefError(book_ref=self.normal())
else:
raise InputError("Can not pad a schema node ref")
raise InputError("Cannot pad a schema node ref.")

d = self._core_dict()
if self.is_talmud():
Expand Down
4 changes: 3 additions & 1 deletion sefaria/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,9 @@ def index_all(cls, index_name, debug=False, for_es=True, action=None):
total_versions = len(versions)
versions = None # release RAM
for title, vlist in list(versions_by_index.items()):
cls.curr_index = vlist[0].get_index() if len(vlist) > 0 else None
if len(vlist) == 0:
continue
cls.curr_index = vlist[0].get_index()
if for_es:
cls._bulk_actions = []
try:
Expand Down
11 changes: 11 additions & 0 deletions sefaria/system/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,14 @@ def __init__(self, method):
self.method = method
self.message = f"'{method}' is not a valid HTTP API method."
super().__init__(self.message)


class ComplexBookLevelRefError(InputError):
def __init__(self, book_ref):
self.book_ref = book_ref
self.message = (f"You passed '{book_ref}', please pass a more specific ref for this book, and try again. "
f"'{book_ref}' is a \'complex\' book-level ref. We only support book-level "
f"refs in cases of texts with a 'simple' structure. To learn more about the "
f"structure of a text on Sefaria, "
f"see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text")
super().__init__(self.message)
8 changes: 6 additions & 2 deletions sefaria/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
from sefaria.datatype.jagged_array import JaggedTextArray
# noinspection PyUnresolvedReferences
from sefaria.system.exceptions import InputError, NoVersionFoundError
from api.api_errors import APIInvalidInputException
from sefaria.system.database import db
from sefaria.system.decorators import catch_error_as_http
from sefaria.utils.hebrew import has_hebrew, strip_nikkud
Expand Down Expand Up @@ -339,7 +340,10 @@ def find_refs_report_api(request):
@api_view(["POST"])
def find_refs_api(request):
from sefaria.helper.linker import make_find_refs_response
return jsonResponse(make_find_refs_response(request))
try:
return jsonResponse(make_find_refs_response(request))
except APIInvalidInputException as e:
return e.to_json_response()


@api_view(["GET"])
Expand Down Expand Up @@ -470,7 +474,7 @@ def bulktext_api(request, refs):
g = lambda x: request.GET.get(x, None)
min_char = int(g("minChar")) if g("minChar") else None
max_char = int(g("maxChar")) if g("maxChar") else None
res = bundle_many_texts(refs, g("useTextFamily"), g("asSizedString"), min_char, max_char, g("transLangPref"), g("ven"), g("vhe"))
res = bundle_many_texts(refs, int(g("useTextFamily")), g("asSizedString"), min_char, max_char, g("transLangPref"), g("ven"), g("vhe"))
resp = jsonResponse(res, cb)
return resp

Expand Down
22 changes: 18 additions & 4 deletions static/css/s2.css
Original file line number Diff line number Diff line change
Expand Up @@ -7107,6 +7107,11 @@ But not to use a display block directive that might break continuous mode for ot
flex: 1 1 auto;
}

.toolsButton ::before {
position: relative;
top: 3px;
}

.toolsSecondaryButton {
color: var(--dark-grey);
--english-font: var(--english-sans-serif-font-family);
Expand Down Expand Up @@ -11525,6 +11530,19 @@ cursor: pointer;
color: white;
background-color: #18345d;
}
.resourcesLink.studyCompanion {
margin-inline-start: 10px;
}
@media screen and (max-width: 900px) {
.resourcesLink.studyCompanion {
margin-inline-start: 0;
}
}
@media screen and (max-width: 900px) {
.resourcesLink.studyCompanion {
margin-inline-start: 0;
}
}
.resourcesLink.blue img {
filter: invert(1);
opacity: 1;
Expand Down Expand Up @@ -13539,10 +13557,6 @@ span.ref-link-color-3 {color: blue}

}

.productsDevBox p {
margin-top: 0;
}

.productsDevBox a::after {
content: " ›";
color: var(--commentary-blue);
Expand Down
4 changes: 4 additions & 0 deletions static/font-awesome/css/font-awesome.css
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,10 @@
.fa-twitter:before {
content: "\f099";
}
.fa-X:before {
content: "𝕏";
font-size: larger;
}
.fa-facebook-f:before,
.fa-facebook:before {
content: "\f09a";
Expand Down
Loading

0 comments on commit affb67e

Please sign in to comment.