Merge remote-tracking branch 'origin/master' into failing-tests

.github/workflows/continuous.yaml

@@ -1,11 +1,7 @@
 name: Continuous
 on:
   push:
-    branches:
-      - "*"
   pull_request:
-    branches:
-      - "*"
 
 concurrency:
   group: ${{ github.ref }}

.github/workflows/production-deploy.yaml

@@ -220,7 +220,6 @@ jobs:
           IMAGE_NAME: "${{ secrets.IMAGE_NAME }}"
           CHART_VERSION: "${{ steps.chart_version.outputs.chart_version }}"
       - name: Update workflow default chart
-        if: github.ref == 'refs/heads/master'
         run: > 
           curl -L 
           -X PATCH 

helm-chart/sefaria-project/templates/configmap/mongo-destroy.yaml

@@ -8,7 +8,7 @@ metadata:
     {{- include "sefaria.labels" . | nindent 4 }}
   annotations:
     helm.sh/hook: post-delete
-    helm.sh/hook-delete-policy: hook-succeeded
+    helm.sh/hook-delete-policy: hook-succeeded, hook-failed
     helm.sh/hook-weight: "5"
 data:
   destroy-mongo.sh: |-

reader/views.py

@@ -1215,7 +1215,6 @@ def edit_text(request, ref=None, lang=None, version=None):
                 mode = "Add"
             else:
                 # Pull a particular section to edit
-                version = version.replace("_", " ") if version else None
                 #text = get_text(ref, lang=lang, version=version)
                 text = TextFamily(Ref(ref), lang=lang, version=version).contents()
                 text["mode"] = request.path.split("/")[1]
@@ -1389,15 +1388,11 @@ def texts_api(request, tref):
         commentary = bool(int(request.GET.get("commentary", False)))
         pad        = bool(int(request.GET.get("pad", 1)))
         versionEn  = request.GET.get("ven", None)
+        versionHe  = request.GET.get("vhe", None)
         firstAvailableRef = bool(int(request.GET.get("firstAvailableRef", False)))  # use first available ref, which may not be the same as oref
         if firstAvailableRef:
             temp_oref = oref.first_available_section_ref()
             oref = temp_oref or oref  # don't overwrite oref if first available section ref fails
-        if versionEn:
-            versionEn = versionEn.replace("_", " ")
-        versionHe  = request.GET.get("vhe", None)
-        if versionHe:
-            versionHe = versionHe.replace("_", " ")
         layer_name = request.GET.get("layer", None)
         alts       = bool(int(request.GET.get("alts", True)))
         wrapLinks = bool(int(request.GET.get("wrapLinks", False)))
@@ -1553,9 +1548,6 @@ def social_image_api(request, tref):
         ref = Ref(tref)
         ref_str = ref.normal() if lang == "en" else ref.he_normal()
 
-        if version:
-            version = version.replace("_", " ")
-
         tf = TextFamily(ref, stripItags=True, lang=lang, version=version, context=0, commentary=False).contents()
 
         he = tf["he"] if type(tf["he"]) is list else [tf["he"]]
@@ -3103,14 +3095,14 @@ def add_new_topic_api(request):
         data = json.loads(request.POST["json"])
         isTopLevelDisplay = data["category"] == Topic.ROOT
         t = Topic({'slug': "", "isTopLevelDisplay": isTopLevelDisplay, "data_source": "sefaria", "numSources": 0})
-        update_topic_titles(t, data)
+        update_topic_titles(t, **data)
 
         if not isTopLevelDisplay:  # not Top Level so create an IntraTopicLink to category
             new_link = IntraTopicLink({"toTopic": data["category"], "fromTopic": t.slug, "linkType": "displays-under", "dataSource": "sefaria"})
             new_link.save()
 
         if data["category"] == 'authors':
-            t = update_authors_place_and_time(t, data)
+            t = update_authors_place_and_time(t, **data)
 
         t.description_published = True
         t.data_source = "sefaria"  # any topic edited manually should display automatically in the TOC and this flag ensures this
@@ -3165,15 +3157,15 @@ def topics_api(request, topic, v2=False):
         if not request.user.is_staff:
             return jsonResponse({"error": "Adding topics is locked.<br><br>Please email [email protected] if you believe edits are needed."})
         topic_data = json.loads(request.POST["json"])
-        topic_obj = Topic().load({'slug': topic_data["origSlug"]})
+        topic = Topic().load({'slug': topic_data["origSlug"]})
         topic_data["manual"] = True
         author_status_changed = (topic_data["category"] == "authors") ^ (topic_data["origCategory"] == "authors")
-        topic_obj = update_topic(topic_obj, **topic_data)
+        topic = update_topic(topic, **topic_data)
         if author_status_changed:
             library.build_topic_auto_completer()
 
         def protected_index_post(request):
-            return jsonResponse(topic_obj.contents())
+            return jsonResponse(topic.contents())
         return protected_index_post(request)
 
 
@@ -3293,6 +3285,16 @@ def recommend_topics_api(request, ref_list=None):
     return response
 
 
+@api_view(["GET"])
+@catch_error_as_json
+def portals_api(request, slug):
+    """
+    API to get data for a Portal object by slug
+    """
+    portal = Portal.init(slug)
+    return jsonResponse(portal.contents(), callback=request.GET.get("callback", None))
+
+
 @ensure_csrf_cookie
 @sanitize_get_params
 def global_activity(request, page=1):
@@ -4012,8 +4014,17 @@ def random_text_api(request):
     """
     Return Texts API data for a random ref.
     """
-    categories = set(request.GET.get('categories', '').split('|'))
-    titles = set(request.GET.get('titles', '').split('|'))
+
+    if "categories" in request.GET:
+        categories = set(request.GET.get('categories', '').split('|'))
+    else:
+        categories = None
+
+    if "titles" in request.GET:
+        titles = set(request.GET.get('titles', '').split('|'))
+    else:
+        titles = None
+
     response = redirect(iri_to_uri("/api/texts/" + random_ref(categories, titles)) + "?commentary=0&context=0", permanent=False)
     return response
 
@@ -4565,4 +4576,3 @@ def isNodeJsReachable():
         logger.warn("Failed rollout healthcheck. Healthcheck Response: {}".format(resp))
 
     return http.JsonResponse(resp, status=statusCode)
-

requirements.txt

@@ -64,6 +64,8 @@ user-agents==2.2.0
 sentry-sdk==1.26.0
 babel
 python-bidi
+requests
+Cerberus
 
 opentelemetry-distro
 opentelemetry-exporter-otlp

scripts/migrations/add_topic_images.py

@@ -0,0 +1,40 @@
+import django
+
+django.setup()
+
+from sefaria.helper.topic import add_image_to_topic
+
+## Adding images
+
+hardcodedTopicImagesMap = {
+    'rosh-hashanah': {'image_uri': 'https://storage.googleapis.com/img.sefaria.org/topics/rosh-hashanah.jpeg',
+                      'enCaption': 'Rosh Hashanah, Arthur Szyk (1894-1951) Tempera and ink on paper. New Canaan, 1948. Collection of Yeshiva University Museum. Gift of Charles Frost',
+                      'heCaption': 'ראש השנה, ארתור שיק, ארה״ב 1948. אוסף ישיבה יוניברסיטי'},
+
+    'yom-kippur': {'image_uri': 'https://storage.googleapis.com/img.sefaria.org/topics/yom-kippur.jpeg',
+                   'enCaption': 'Micrography of Jonah being swallowed by the fish. Germany, 1300-1500, The British Library',
+                   'heCaption': 'מיקרוגרפיה של יונה בבטן הדג, מתוך ספר יונה ההפטרה של יום כיפור, 1300-1500'},
+
+    'the-four-species': {'image_uri': 'https://storage.googleapis.com/img.sefaria.org/topics/the-four-species.jpg',
+                         'enCaption': 'Etrog container, K B, late 19th century, Germany. The Jewish Museum, Gift of Dr. Harry G. Friedman',
+                         'heCaption': 'תיבת אתרוג, סוף המאה ה19, גרמניה. המוזיאון היהודי בניו יורק, מתנת דר. הארי ג. פרידמן  '},
+
+    'sukkot': {'image_uri': 'https://storage.googleapis.com/img.sefaria.org/topics/sukkot.jpg',
+               'enCaption': 'Detail of a painting of a sukkah. Image taken from f. 316v of Forli Siddur. 1383, Italian rite. The British Library',
+               'heCaption': 'פרט ציור של סוכה עם שולחן פרוש ושלוש דמויות. דימוי מתוך סידור פורלי, 1383 איטליה'},
+
+    'simchat-torah': {'image_uri': 'https://storage.googleapis.com/img.sefaria.org/topics/simchat-torah.jpg',
+                      'enCaption': 'Rosh Hashanah postcard: Hakafot, Haim Yisroel Goldberg (1888-1943) Publisher: Williamsburg Post Card Co. Germany, ca. 1915 Collection of Yeshiva University Museum',
+                      'heCaption': 'גלויה לראש השנה: הקפות, חיים גולדברג, גרמניה 1915, אוסף ישיבה יוניברסיטי'},
+
+    'shabbat': {'image_uri': 'https://storage.googleapis.com/img.sefaria.org/topics/shabbat.jpg',
+                'enCaption': 'Friday Evening, Isidor Kaufmann, Austria c. 1920. The Jewish Museum, Gift of Mr. and Mrs. M. R. Schweitzer',
+                'heCaption': 'שישי בערב, איזידור קאופמן, וינה 1920. המוזיאון היהודי בניו יורק, מתנת  מר וגברת מ.ר. שוויצר'},
+
+}
+
+for topic in hardcodedTopicImagesMap:
+    add_image_to_topic(topic,
+                       image_uri=hardcodedTopicImagesMap[topic]["image_uri"],
+                       en_caption=hardcodedTopicImagesMap[topic]["enCaption"],
+                       he_caption=hardcodedTopicImagesMap[topic]["heCaption"])

sefaria/helper/tests/topic_test.py

@@ -52,10 +52,10 @@ def grandchild_of_root_with_self_link(child_of_root_with_self_link):
 
 
 @pytest.fixture(autouse=True, scope='module')
-def root_wout_self_link():
-	# create second branch of tree starting with root_wout_self_link
+def author_root():
+	# create second branch of tree starting with author_root
 	t = Topic({'slug': "", "isTopLevelDisplay": True, "data_source": "sefaria", "numSources": 0})
-	title = "Normal Root"
+	title = "Authors"
 	he_title = title[::-1]
 	t.add_primary_titles(title, he_title)
 	t.set_slug_to_primary_title()
@@ -66,73 +66,86 @@ def root_wout_self_link():
 
 
 @pytest.fixture(autouse=True, scope='module')
-def child_of_root_wout_self_link(root_wout_self_link):
+def actual_author(author_root):
 	t = Topic({'slug': "", "isTopLevelDisplay": False, "data_source": "sefaria", "numSources": 0})
-	title = "Normal Root Leaf Node"
+	title = "Author Dude"
 	he_title = title[::-1]
 	t.add_primary_titles(title, he_title)
 	t.set_slug_to_primary_title()
 	t.save()
 	l = IntraTopicLink({"linkType": "displays-under", "fromTopic": t.slug,
-						"toTopic": root_wout_self_link["topic"].slug, "dataSource": "sefaria",
-						"class": "intraTopic"}).save()  # root_wout_self_link has child leaf_node
+						"toTopic": author_root["topic"].slug, "dataSource": "sefaria",
+						"class": "intraTopic"}).save()  # author_root has child leaf_node
 	yield {"topic": t, "link": l}
 	t.delete()
 	l.delete()
 
 
-def test_title_and_desc(root_wout_self_link, child_of_root_wout_self_link, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link):
-	for t in [root_wout_self_link, child_of_root_wout_self_link, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link]:
-		new_values = {"title": "new title", "heTitle": "new hebrew title", "description": {"en": "desc", "he": "hebrew desc"}}
+def test_title_and_desc(author_root, actual_author, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link):
+	for count, t in enumerate([author_root, actual_author, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link]):
+		new_values = {"title": f"new title {count+1}",
+					  "altTitles": {"en": [f"New Alt title {count+1}"], "he": [f"New He Alt Title {count+1}"]},
+					  "heTitle": f"new hebrew title {count+1}", "description": {"en": f"new desc", "he": "new hebrew desc"}}
 		update_topic(t["topic"], **new_values)
 		assert t["topic"].description == new_values["description"]
-		assert t["topic"].get_primary_title('en') == new_values['title']
 		assert t["topic"].get_primary_title('he') == new_values['heTitle']
-
-
-def test_change_categories_and_titles(root_wout_self_link, root_with_self_link):
+		assert t["topic"].get_titles('en') == [new_values["title"]]+new_values["altTitles"]['en']
+
+def test_author_root(author_root, actual_author):
+	new_values = {"category": "authors", "title": actual_author["topic"].get_primary_title('en'),
+				  "heTitle": actual_author["topic"].get_primary_title('he'),
+				  "birthPlace": "Kyoto, Japan", "birthYear": 1300}
+	assert Place().load({'key': new_values["birthPlace"]}) is None
+	update_topic(actual_author["topic"], **new_values)
+	assert Place().load({'key': new_values["birthPlace"]})
+	assert actual_author["topic"].properties["birthYear"]["value"] == 1300
+	Place().load({'key': new_values["birthPlace"]}).delete()
+
+def test_change_categories_and_titles(author_root, root_with_self_link):
 	# tests moving both root categories down the tree and back up and asserting that moving down the tree changes the tree
 	# and assert that moving it back to the root position yields the original tree.
-	# also tests
-
-	orig_tree_from_normal_root = library.get_topic_toc_json_recursive(root_wout_self_link["topic"])
+	orig_tree_from_normal_root = library.get_topic_toc_json_recursive(author_root["topic"])
 	orig_tree_from_root_with_self_link = library.get_topic_toc_json_recursive(root_with_self_link["topic"])
 	orig_trees = [orig_tree_from_normal_root, orig_tree_from_root_with_self_link]
-	roots = [root_wout_self_link["topic"], root_with_self_link["topic"]]
+	roots = [author_root["topic"], root_with_self_link["topic"]]
 	orig_titles = [roots[0].get_primary_title('en'), roots[1].get_primary_title('en')]
+	orig_he_titles = [roots[0].get_primary_title('he'), roots[1].get_primary_title('he')]
 	for i, root in enumerate(roots):
 		other_root = roots[1 - i]
-		update_topic(root, title=f"fake new title {i+1}", category=other_root.slug)  # move root to be child of other root
+		update_topic(root, title=f"fake new title {i+1}", heTitle=f"fake new he title {i+1}", category=other_root.slug)  # move root to be child of other root
 		new_tree = library.get_topic_toc_json_recursive(other_root)
 		assert new_tree != orig_trees[i]  # assert that the changes in the tree have occurred
-		assert root.get_primary_title('en') != orig_titles[i]
-		update_topic(root, title=orig_titles[i], category=Topic.ROOT)  # move it back to the main menu
-		assert root.get_primary_title('en') == orig_titles[i]
+		assert root.get_titles('en') != [orig_titles[i]]
+		assert root.get_titles('he') != [orig_he_titles[i]]
+		update_topic(root, title=orig_titles[i], heTitle=orig_he_titles[i], category=Topic.ROOT)  # move it back to the main menu
+		assert root.get_titles('en') == [orig_titles[i]]
+		assert root.get_titles('he') == [orig_he_titles[i]]
+
 
 	final_tree_from_normal_root = library.get_topic_toc_json_recursive(roots[0])
 	final_tree_from_root_with_self_link = library.get_topic_toc_json_recursive(roots[1])
 	assert final_tree_from_normal_root == orig_tree_from_normal_root  # assert that the tree is back to normal
 	assert final_tree_from_root_with_self_link == orig_tree_from_root_with_self_link
 
 
-def test_change_categories(root_wout_self_link, child_of_root_wout_self_link, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link):
+def test_change_categories(author_root, actual_author, root_with_self_link, child_of_root_with_self_link, grandchild_of_root_with_self_link):
 	# tests moving topics across the tree to a different root
 
-	orig_tree_from_normal_root = library.get_topic_toc_json_recursive(root_wout_self_link["topic"])
+	orig_tree_from_normal_root = library.get_topic_toc_json_recursive(author_root["topic"])
 	orig_tree_from_root_with_self_link = library.get_topic_toc_json_recursive(root_with_self_link["topic"])
 
-	topic_change_category(child_of_root_with_self_link["topic"], root_wout_self_link["topic"].slug)
-	topic_change_category(child_of_root_wout_self_link["topic"], root_with_self_link["topic"].slug)
+	topic_change_category(child_of_root_with_self_link["topic"], author_root["topic"].slug)
+	topic_change_category(actual_author["topic"], root_with_self_link["topic"].slug)
 
-	new_tree_from_normal_root = library.get_topic_toc_json_recursive(root_wout_self_link["topic"])
+	new_tree_from_normal_root = library.get_topic_toc_json_recursive(author_root["topic"])
 	new_tree_from_root_with_self_link = library.get_topic_toc_json_recursive(root_with_self_link["topic"])
 	assert new_tree_from_normal_root != orig_tree_from_normal_root
 	assert new_tree_from_root_with_self_link != orig_tree_from_root_with_self_link
 
 	topic_change_category(child_of_root_with_self_link["topic"], root_with_self_link["topic"].slug)
-	topic_change_category(child_of_root_wout_self_link["topic"], root_wout_self_link["topic"].slug)
+	topic_change_category(actual_author["topic"], author_root["topic"].slug)
 
-	new_tree_from_normal_root = library.get_topic_toc_json_recursive(root_wout_self_link["topic"])
+	new_tree_from_normal_root = library.get_topic_toc_json_recursive(author_root["topic"])
 	new_tree_from_root_with_self_link = library.get_topic_toc_json_recursive(root_with_self_link["topic"])
 	assert new_tree_from_normal_root == orig_tree_from_normal_root
 	assert new_tree_from_root_with_self_link == orig_tree_from_root_with_self_link