Merge branch 'modularization-main' into bug/sc-28503/search-filtering…

…-bug-for-sheets

build/ci/sandbox-values.yaml

@@ -44,7 +44,7 @@ monitor:
     tag: 
 secrets:
   localSettings: 
-    ref: local-settings-secrets-dev
+    ref: local-settings-secrets
   backupManager: 
     ref: backup-manager
   slackWebhook: 

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -47,6 +47,7 @@ data:
     ADMINS = (
         ('Sefaria Developers', '[email protected]'),
     )
+    ADMIN_PATH = os.getenv("SEFARIA_ADMIN_PATH")
 
     MANAGERS = ADMINS
 

reader/views.py

@@ -78,7 +78,7 @@
 from sefaria.utils.user import delete_user_account
 from django.core.mail import EmailMultiAlternatives
 from babel import Locale
-from sefaria.helper.topic import update_topic, update_topic_titles
+from sefaria.helper.topic import update_topic
 from sefaria.helper.category import update_order_of_category_children, check_term
 
 if USE_VARNISH:
@@ -1594,8 +1594,38 @@ def parashat_hashavua_api(request):
     p.update(TextFamily(Ref(p["ref"])).contents())
     return jsonResponse(p, callback)
 
+def find_holiday_in_hebcal_results(response):
+    for hebcal_holiday in json.loads(response.text)['items']:
+        if hebcal_holiday['category'] != 'holiday':
+            continue
+        for result in get_name_completions(hebcal_holiday['hebrew'], 10, False)['completion_objects']:
+            if result['type'] == 'Topic':
+                topic = Topic.init(result['key'])
+                if topic:
+                    return topic.contents()
+    return null
 
 @catch_error_as_json
+def next_holiday(request):
+    from datetime import datetime
+    from dateutil.relativedelta import relativedelta
+    import requests
+    current_date = datetime.now().date()
+    date_in_three_months = current_date + relativedelta(months=+3)
+
+    # Format the date as YYYY-MM-DD
+    current_date = current_date.strftime('%Y-%m-%d')
+    date_in_three_months = date_in_three_months.strftime("%Y-%m-%d")
+    response = requests.get(f"https://www.hebcal.com/hebcal?v=1&cfg=json&maj=on&start={current_date}&end={date_in_three_months}")
+    if response.status_code == 200:
+        topic = find_holiday_in_hebcal_results(response)
+        if topic:
+            return jsonResponse(topic)
+        else:
+            return jsonResponse({"error": "Couldn't find any topics corresponding to HebCal results"})
+    else:
+        return jsonResponse({"error": "Couldn't establish connection with HebCal API"})
+@catch_error_as_json
 def table_of_contents_api(request):
     return jsonResponse(library.get_toc(), callback=request.GET.get("callback", None))
 
@@ -2438,9 +2468,9 @@ def _internal_do_post(request, update, cat, uid, **kwargs):
         else:
             return jsonResponse({"error": "Only Sefaria Moderators can add or delete categories."})
 
-        j = request.POST.get("json")
+        j = request.body
         if not j:
-            return jsonResponse({"error": "Missing 'json' parameter in post data."})
+            return jsonResponse({"error": "Missing data in POST request."})
         j = json.loads(j)
         update = int(request.GET.get("update", False))
         new_category = Category().load({"path": j["path"]})
@@ -2486,6 +2516,18 @@ def _internal_do_post(request, update, cat, uid, **kwargs):
 
     return jsonResponse({"error": "Unsupported HTTP method."})
 
+@catch_error_as_json
+@csrf_exempt
+def parasha_data_api(request):
+    from sefaria.utils.calendars import make_parashah_response_from_calendar_entry
+    diaspora = request.GET.get("diaspora", "1")
+    datetime_obj = timezone.localtime(timezone.now())
+    if diaspora not in ["0", "1"]:
+        return jsonResponse({"error": "'Diaspora' parameter must be 1 or 0."})
+    else:
+        diaspora = True if diaspora == "1" else False
+        db_parasha = get_parasha(datetime_obj, diaspora=diaspora)
+        return jsonResponse(make_parashah_response_from_calendar_entry(db_parasha, include_topic_slug=True)[0])
 
 @catch_error_as_json
 @csrf_exempt
@@ -3109,7 +3151,9 @@ def add_new_topic_api(request):
         data = json.loads(request.POST["json"])
         isTopLevelDisplay = data["category"] == Topic.ROOT
         t = Topic({'slug': "", "isTopLevelDisplay": isTopLevelDisplay, "data_source": "sefaria", "numSources": 0})
-        update_topic_titles(t, **data)
+        titles = data.get('titles')
+        if titles:
+            t.set_titles(titles)
         t.set_slug_to_primary_title()
         if not isTopLevelDisplay:  # not Top Level so create an IntraTopicLink to category
             new_link = IntraTopicLink({"toTopic": data["category"], "fromTopic": t.slug, "linkType": "displays-under", "dataSource": "sefaria"})
@@ -4027,15 +4071,13 @@ def digitized_by_sefaria(request):
         "texts": texts,
     })
 
-
 def parashat_hashavua_redirect(request):
     """ Redirects to this week's Parashah"""
     diaspora = request.GET.get("diaspora", "1")
     calendars = get_keyed_calendar_items()  # TODO Support israel / customs
     parashah = calendars["Parashat Hashavua"]
     return redirect(iri_to_uri("/" + parashah["url"]), permanent=False)
 
-
 def daf_yomi_redirect(request):
     """ Redirects to today's Daf Yomi"""
     calendars = get_keyed_calendar_items()

requirements.txt

@@ -1,15 +1,18 @@
 Appium-Python-Client==1.2.0
-apscheduler==3.6.
+Cerberus
+PyJWT==1.7.1 # pinned b/c current version 2.0.0 breaks simplejwt. waiting for 2.0.1
+apscheduler==3.6.*
+babel
 bleach==1.4.2
 boto3==1.16.6
 bs4==0.0.1
+celery[redis]
 convertdate==2.2.2
 cython==0.29.14
 dateutils==0.6.12
 datrie==0.8.2
 deepdiff==3.3.0
 diff_match_patch==20200713
-django_mobile==0.7.0
 django-anymail==7.2.*
 django-debug-toolbar==2.2 # not used in prod
 django-recaptcha==2.0.6
@@ -18,18 +21,22 @@ django-structlog==1.6.2
 django-user-agents==0.4.0
 django-webpack-loader==1.4.1
 django==1.11.*
+django_mobile==0.7.0
 djangorestframework @ https://github.com/encode/django-rest-framework/archive/3.11.1.tar.gz
 djangorestframework_simplejwt==3.3.0
-PyJWT==1.7.1 # pinned b/c current version 2.0.0 breaks simplejwt. waiting for 2.0.1
+dnspython~=2.5.0
 elasticsearch==8.8.2
-git+https://github.com/Sefaria/[email protected]#egg=elasticsearch-dsl
-git+https://github.com/Sefaria/[email protected]#egg=sefaria_llm_interface&subdirectory=app/llm_interface
 geojson==2.5.0
 geopy==2.3.0
 gevent==20.12.0; sys_platform != 'darwin'
+git+https://github.com/Sefaria/[email protected]#egg=sefaria_llm_interface&subdirectory=app/llm_interface
+git+https://github.com/Sefaria/[email protected]#egg=elasticsearch-dsl
 google-api-python-client==1.12.5
+google-auth-oauthlib==0.4.2
+google-auth==1.24.0
 google-cloud-logging==1.15.1
 google-cloud-storage==1.32.0
+google-re2
 gunicorn==20.0.4
 html5lib==0.9999999
 httplib2==0.18.1
@@ -38,37 +45,31 @@ jedi==0.18.1 # Ipython was previosuly pinned at 7.18 because Jedi 0.18 broke it.
 jsonpickle==1.4.1
 lxml==4.6.1
 mailchimp==2.0.9
-google-auth==1.24.0
-google-auth-oauthlib==0.4.2
 p929==0.6.1
 pathos==0.2.6
-pillow==8.0.1; sys_platform == 'linux'
 pillow==10.0.1; sys_platform != 'linux'
-psycopg2==2.8.6
+pillow==8.0.1; sys_platform == 'linux'
+psycopg2==2.8.6 #for dev: psycopg2-binary==2.8.6
 py2-py3-django-email-as-username==1.7.1
 pymongo==3.12.*
 pytest==6.1.1
+python-bidi
 pytz
 pyyaml==6.0.1
 rauth==0.7.3
 redis==3.5.3
 regex==2020.10.23
+requests
 roman==3.3
 selenium==3.141.0
+sentry-sdk==1.26.0
 tqdm==4.51.0
 ua-parser==0.10.0
 undecorated==0.3.0
 unicodecsv==0.14.1
 unidecode==1.1.1
 user-agents==2.2.0
-sentry-sdk==1.26.0
-babel
-python-bidi
-requests
-Cerberus
-celery[redis]
-google-re2
-dnspython~=2.5.0
+
 
 #opentelemetry-distro
 #opentelemetry-exporter-otlp

sefaria/helper/category.py

@@ -186,14 +186,15 @@ def update_order_of_category_children(cat, uid, subcategoriesAndBooks):
     results = []
     for subcategoryOrBook in subcategoriesAndBooks:
         order += 5
-        try:
-            obj = library.get_index(subcategoryOrBook).contents(raw=True)
-            obj['order'] = [order]
-            result = tracker.update(uid, Index, obj)
-        except BookNameError as e:
-            obj = Category().load({"path": cat_path+[subcategoryOrBook]}).contents()
-            obj['order'] = order
-            result = tracker.update(uid, Category, obj)
+        book = Index().load({'title': subcategoryOrBook, 'categories': cat_path})
+        if book:
+            book = book.contents(raw=True)
+            book['order'] = [order]
+            result = tracker.update(uid, Index, book)
+        else:
+            cat = Category().load({"path": cat_path+[subcategoryOrBook]}).contents()
+            cat['order'] = order
+            result = tracker.update(uid, Category, cat)
         results.append(result.contents())
     return results
 

sefaria/helper/tests/topic_test.py

@@ -67,6 +67,17 @@ def author_root():
 	yield {"topic": t, "link": l}
 	t.delete()
 
+@pytest.fixture(autouse=True, scope='module')
+def some_topic():
+	t = Topic({'slug': "abcd_test", "data_source": "sefaria", "numSources": 0})
+	title = "title in English"
+	he_title = "כותרת בעברית"
+	t.add_primary_titles(title, he_title)
+	t.set_slug_to_primary_title()
+	t.save()
+	yield t
+	t.delete()
+
 
 @pytest.fixture(autouse=True, scope='module')
 def actual_author(author_root):
@@ -86,18 +97,24 @@ def actual_author(author_root):
 
 def test_title_and_desc(author_root, actual_author, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link):
 	for count, t in enumerate([author_root, actual_author, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link]):
-		new_values = {"title": f"new title {count+1}",
-					  "altTitles": {"en": [f"New Alt title {count+1}"], "he": [f"New He Alt Title {count+1}"]},
-					  "heTitle": f"new hebrew title {count+1}", "description": {"en": f"new desc", "he": "new hebrew desc"}}
+		en_primary_title = {"text": f"new title {count+1}", "primary": True, "lang": 'en'}
+		he_primary_title = {"lang": "he", "text": f"new hebrew title {count+1}", "primary": True}
+
+		en_alt_title = {"lang": "en",  "text": f"New Alt title {count+1}"}
+		he_alt_title = {"lang": "he", "text": f"New He Alt Title {count+1}"}
+
+		new_values = {"titles": [en_primary_title, en_alt_title, he_alt_title, he_primary_title],
+					  "description": {"en": f"new desc", "he": "new hebrew desc"}}
 		topic.update_topic(t["topic"], **new_values)
 		assert t["topic"].description == new_values["description"]
-		assert t["topic"].get_primary_title('he') == new_values['heTitle']
-		assert t["topic"].get_titles('en') == [new_values["title"]]+new_values["altTitles"]['en']
+		assert t["topic"].get_primary_title('he') == he_primary_title['text']
+		assert t["topic"].get_titles('en') == [en_primary_title['text'], en_alt_title['text']]
 
 def test_author_root(author_root, actual_author):
-	new_values = {"category": "authors", "title": actual_author["topic"].get_primary_title('en'),
-				  "heTitle": actual_author["topic"].get_primary_title('he'),
-				  "birthPlace": "Kyoto, Japan", "birthYear": 1300}
+	new_values = {"category": "authors", "titles": [
+		{'text': actual_author["topic"].get_primary_title('en'), "lang": 'en', 'primary': True},
+		{"text": actual_author["topic"].get_primary_title('he'), "lang": 'he', 'primary': True}],
+	  	"birthPlace": "Kyoto, Japan", "birthYear": 1300}
 	assert Place().load({'key': new_values["birthPlace"]}) is None
 	topic.update_topic(actual_author["topic"], **new_values)
 	assert Place().load({'key': new_values["birthPlace"]})
@@ -111,18 +128,19 @@ def test_change_categories_and_titles(author_root, root_with_self_link):
 	orig_tree_from_root_with_self_link = library.get_topic_toc_json_recursive(root_with_self_link["topic"])
 	orig_trees = [orig_tree_from_normal_root, orig_tree_from_root_with_self_link]
 	roots = [author_root["topic"], root_with_self_link["topic"]]
-	orig_titles = [roots[0].get_primary_title('en'), roots[1].get_primary_title('en')]
-	orig_he_titles = [roots[0].get_primary_title('he'), roots[1].get_primary_title('he')]
+	orig_titles = [{'text': roots[0].get_primary_title('en'), 'lang':'en', 'primary': True}, {'text': roots[1].get_primary_title('en'), 'lang':'en', 'primary': True}]
+	orig_he_titles = [{'text': roots[0].get_primary_title('he'), 'lang':'he', 'primary': True}, {'text': roots[1].get_primary_title('he'), 'lang':'he', 'primary': True}]
 	for i, root in enumerate(roots):
 		other_root = roots[1 - i]
-		topic.update_topic(root, title=f"fake new title {i+1}", heTitle=f"fake new he title {i+1}", category=other_root.slug)  # move root to be child of other root
+		topic.update_topic(root, titles=[{'text': f"fake new title {i+1}", 'lang': 'he', 'primary': True},
+										 {'text': f"fake new he title {i+1}", 'lang': 'he', 'primary': True}], category=other_root.slug)  # move root to be child of other root
 		new_tree = library.get_topic_toc_json_recursive(other_root)
 		assert new_tree != orig_trees[i]  # assert that the changes in the tree have occurred
-		assert root.get_titles('en') != [orig_titles[i]]
-		assert root.get_titles('he') != [orig_he_titles[i]]
-		topic.update_topic(root, title=orig_titles[i], heTitle=orig_he_titles[i], category=Topic.ROOT)  # move it back to the main menu
-		assert root.get_titles('en') == [orig_titles[i]]
-		assert root.get_titles('he') == [orig_he_titles[i]]
+		assert root.get_titles('en') != [orig_titles[i]['text']]
+		assert root.get_titles('he') != [orig_he_titles[i]['text']]
+		topic.update_topic(root, titles=[orig_titles[i], orig_he_titles[i]], category=Topic.ROOT)  # move it back to the main menu
+		assert root.get_titles('en') == [orig_titles[i]['text']]
+		assert root.get_titles('he') == [orig_he_titles[i]['text']]
 
 
 	final_tree_from_normal_root = library.get_topic_toc_json_recursive(roots[0])
@@ -178,3 +196,19 @@ def test_calculate_approved_review_state(current, requested, was_ai_generated, m
 ])
 def test_get_merged_descriptions(current, requested, merged):
 	assert topic._get_merged_descriptions(current, requested) == merged
+
+
+def test_update_topic(some_topic):
+	topic.update_topic(some_topic, titles=[{"text": "Tamar", "lang": "en", "primary": True},
+							 {"text": "תמר", "lang": "he", "primary": True, "disambiguation": "יהודה"}])
+	assert some_topic.titles == [{"text": "Tamar", "lang": "en", "primary": True},
+						 {"text": "תמר", "lang": "he", "primary": True, "disambiguation": "יהודה"}]
+
+	topic.update_topic(some_topic, description={"en": "abcdefg"})
+	assert some_topic.description == {"en": "abcdefg"}
+
+	with pytest.raises(Exception):
+		topic.update_topic(some_topic, titles=[{"a": "Tamar", "b": "en"},
+								 {"c": "תמר", "lang": "d", "disambiguation": "יהודה"}])
+	with pytest.raises(Exception):
+		topic.update_topic(some_topic, slug='abc')