Merge branch 'modularization-main' into feature/sc-27901/card-box-com…

…ponent

build/ci/integration-values.yaml

@@ -57,7 +57,6 @@ localSettings:
   DEBUG: true
   DOMAIN_LANGUAGE: {}
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL:  "http://${NAME}.integration.sefaria.org"
   OFFLINE: false

build/ci/production-values.yaml

@@ -235,7 +235,6 @@ localSettings:
   }
   MONGO_HOST: "mongo"
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL:  "http://www.sefaria.org"
   OFFLINE: false

build/ci/sandbox-values.yaml

@@ -53,7 +53,6 @@ localSettings:
   DEBUG: false
   DOMAIN_LANGUAGE: {}
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL:  "http://${NAME}.cauldron.sefaria.org" 
   OFFLINE: false

build/notify/notifyEnd.js

@@ -21,7 +21,7 @@ console.log(`
 
 const jobKeys = [
     "Jest",
-    "PyTest",
+    "Continuous Testing: PyTest",
     "Playwright",
 ];
 

docs/openAPI.json

@@ -4187,7 +4187,6 @@
                       },
                       "categoryDescription": {},
                       "numSources": 1767,
-                      "good_to_promote": true,
                       "description_published": true,
                       "data_source": "sefaria",
                       "primaryTitle": {
@@ -4591,7 +4590,6 @@
                         "isTopLevelDisplay": true,
                         "displayOrder": 30,
                         "numSources": 2937,
-                        "good_to_promote": true,
                         "primaryTitle": {
                           "en": "Prayer",
                           "he": "תפילה"
@@ -4623,7 +4621,6 @@
                           "_temp_id": "תורה"
                         },
                         "numSources": 2333,
-                        "good_to_promote": true,
                         "primaryTitle": {
                           "en": "Torah",
                           "he": "תורה"
@@ -4766,7 +4763,6 @@
                         },
                         "categoryDescription": {},
                         "numSources": 1967,
-                        "good_to_promote": true,
                         "description_published": true,
                         "data_source": "sefaria",
                         "primaryTitle": {
@@ -4869,7 +4865,6 @@
                           "categoryDescription": {},
                           "displayOrder": 0,
                           "numSources": 1662,
-                          "good_to_promote": true,
                           "description_published": true,
                           "data_source": "sefaria",
                           "image": {
@@ -5162,7 +5157,6 @@
                           }
                         },
                         "numSources": 7,
-                        "good_to_promote": true,
                         "primaryTitle": {
                           "en": "Metushelach",
                           "he": "מתושלח"
@@ -9761,7 +9755,6 @@
               "categoryDescription": {},
               "displayOrder": 0,
               "numSources": 1662,
-              "good_to_promote": true,
               "description_published": true,
               "data_source": "sefaria",
               "image": {
@@ -10071,9 +10064,6 @@
                 "format": "int32",
                 "type": "integer"
               },
-              "good_to_promote": {
-                "type": "boolean"
-              },
               "description_published": {
                 "type": "boolean"
               },
@@ -10089,7 +10079,7 @@
                 }
               }
             },
-            "example": "{\n\"slug\": \"metushelach\",\n\"titles\": [\n{\n\"text\": \"Metushelach\",\n\"lang\": \"en\",\n\"primary\": true,\n\"transliteration\": true\n},\n{\n\"text\": \"מתושלח\",\n\"lang\": \"he\",\n\"primary\": true\n},\n{\n\"text\": \"Methuselah\",\n\"lang\": \"en\"\n},\n{\n\"text\": \"Methushelach\",\n\"lang\": \"en\"\n}\n],\n\"subclass\": \"person\",\n\"alt_ids\": {\n\"_temp_id\": \"מתושלח\",\n\"wikidata\": \"Q156290\"\n},\n\"properties\": {\n\"enWikiLink\": {\n\"value\": \"https://en.wikipedia.org/wiki/Methuselah\",\n\"dataSource\": \"wikidata\"\n},\n\"heWikiLink\": {\n\"value\": \"https://he.wikipedia.org/wiki/מתושלח\",\n\"dataSource\": \"wikidata\"\n},\n\"deWikiLink\": {\n\"value\": \"https://de.wikipedia.org/wiki/Methusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"esWikiLink\": {\n\"value\": \"https://es.wikipedia.org/wiki/Matusalén\",\n\"dataSource\": \"wikidata\"\n},\n\"frWikiLink\": {\n\"value\": \"https://fr.wikipedia.org/wiki/Mathusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"ruWikiLink\": {\n\"value\": \"https://ru.wikipedia.org/wiki/Мафусал_(потомок_Сифа)\",\n\"dataSource\": \"wikidata\"\n}\n},\n\"numSources\": 7,\n\"good_to_promote\": true,\n\"primaryTitle\": {\n\"en\": \"Metushelach\",\n\"he\": \"מתושלח\"\n}\n}"
+            "example": "{\n\"slug\": \"metushelach\",\n\"titles\": [\n{\n\"text\": \"Metushelach\",\n\"lang\": \"en\",\n\"primary\": true,\n\"transliteration\": true\n},\n{\n\"text\": \"מתושלח\",\n\"lang\": \"he\",\n\"primary\": true\n},\n{\n\"text\": \"Methuselah\",\n\"lang\": \"en\"\n},\n{\n\"text\": \"Methushelach\",\n\"lang\": \"en\"\n}\n],\n\"subclass\": \"person\",\n\"alt_ids\": {\n\"_temp_id\": \"מתושלח\",\n\"wikidata\": \"Q156290\"\n},\n\"properties\": {\n\"enWikiLink\": {\n\"value\": \"https://en.wikipedia.org/wiki/Methuselah\",\n\"dataSource\": \"wikidata\"\n},\n\"heWikiLink\": {\n\"value\": \"https://he.wikipedia.org/wiki/מתושלח\",\n\"dataSource\": \"wikidata\"\n},\n\"deWikiLink\": {\n\"value\": \"https://de.wikipedia.org/wiki/Methusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"esWikiLink\": {\n\"value\": \"https://es.wikipedia.org/wiki/Matusalén\",\n\"dataSource\": \"wikidata\"\n},\n\"frWikiLink\": {\n\"value\": \"https://fr.wikipedia.org/wiki/Mathusalem\",\n\"dataSource\": \"wikidata\"\n},\n\"ruWikiLink\": {\n\"value\": \"https://ru.wikipedia.org/wiki/Мафусал_(потомок_Сифа)\",\n\"dataSource\": \"wikidata\"\n}\n},\n\"numSources\": 7,\n\"primaryTitle\": {\n\"en\": \"Metushelach\",\n\"he\": \"מתושלח\"\n}\n}"
           },
           "url": {
             "description": "The `Ref` in a format appropriate for a URL, with spaces replaced with `.` etc. ",
@@ -10135,7 +10125,6 @@
             },
             "categoryDescription": {},
             "numSources": 217,
-            "good_to_promote": true,
             "description_published": true,
             "primaryTitle": {
               "en": "Hillel",
@@ -10170,10 +10159,6 @@
             "description": "A description of the category of this topic",
             "type": "string"
           },
-          "good_to_promote": {
-            "description": "A topic which will be included in our results from the `random-by-topic` endpoint. ",
-            "type": "boolean"
-          },
           "numSources": {
             "description": "The number of text sources associated with a topic. ",
             "type": "integer",
@@ -10222,7 +10207,6 @@
           },
           "categoryDescription": {},
           "numSources": 120,
-          "good_to_promote": true,
           "description_published": true,
           "data_source": "sefaria",
           "primaryTitle": {

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -144,10 +144,6 @@ data:
     SEARCH_INDEX_NAME_TEXT = 'text'  # name of the ElasticSearch index to use
     SEARCH_INDEX_NAME_SHEET = 'sheet'
 
-    TURN_SERVER = os.getenv("TURN_SERVER") #coturn.cauldron.sefaria.org
-    TURN_SECRET= os.getenv("TURN_SECRET")
-    TURN_USER = os.getenv("TURN_USER")
-
     USE_NODE = True
     NODE_HOST = "http://{}:3000".format(nodejsHost)
     NODE_TIMEOUT = 5

helm-chart/sefaria-project/templates/configmap/local-settings.yaml

@@ -9,7 +9,6 @@ data:
   DEBUG: "{{ .Values.localSettings.DEBUG }}"
   DOMAIN_LANGUAGE: {{ .Values.localSettings.DOMAIN_LANGUAGE | toJson | quote }}
   APSCHEDULER_NAME: {{ tpl .Values.localSettings.APSCHEDULER_NAME . | quote }}
-  TURN_SERVER: {{ .Values.localSettings.TURN_SERVER | quote }}
   USE_CLOUDFLARE: "{{ .Values.localSettings.USE_CLOUDFLARE }}"
   FRONT_END_URL: {{ .Values.localSettings.FRONT_END_URL | quote }}
   OFFLINE: "{{ .Values.localSettings.OFFLINE }}"

helm-chart/sefaria-project/values.yaml

@@ -357,8 +357,6 @@ secrets:
     #   SEFARIA_DB_USER:
     #   SEFARIA_DB_PASSWORD:
     #   SEARCH_URL
-    #   TURN_SECRET:
-    #   TURN_USER:
     #   SEFARIA_BOT_API_KEY:
     #   CLOUDFLARE_ZONE:
     #   CLOUDFLARE_EMAIL:
@@ -451,7 +449,6 @@ localSettings:
     # https://www.sefaria.org: english
     # https://www.sefaria.org.il: hebrew
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL: "http://www.sefaria.org"  # Use "http://${ENV_NAME}.cauldron.sefaria.org" in cauldrons
   OFFLINE: "False"

reader/views.py

@@ -4238,7 +4238,7 @@ def random_by_topic_api(request):
     Returns Texts API data for a random text taken from popular topic tags
     """
     cb = request.GET.get("callback", None)
-    random_topic = get_random_topic(good_to_promote=True)
+    random_topic = get_random_topic('torahtab')
     if random_topic is None:
         return random_by_topic_api(request)
     random_source = get_random_topic_source(random_topic)

sefaria/helper/category.py

@@ -186,14 +186,15 @@ def update_order_of_category_children(cat, uid, subcategoriesAndBooks):
     results = []
     for subcategoryOrBook in subcategoriesAndBooks:
         order += 5
-        try:
-            obj = library.get_index(subcategoryOrBook).contents(raw=True)
-            obj['order'] = [order]
-            result = tracker.update(uid, Index, obj)
-        except BookNameError as e:
-            obj = Category().load({"path": cat_path+[subcategoryOrBook]}).contents()
-            obj['order'] = order
-            result = tracker.update(uid, Category, obj)
+        book = Index().load({'title': subcategoryOrBook, 'categories': cat_path})
+        if book:
+            book = book.contents(raw=True)
+            book['order'] = [order]
+            result = tracker.update(uid, Index, book)
+        else:
+            cat = Category().load({"path": cat_path+[subcategoryOrBook]}).contents()
+            cat['order'] = order
+            result = tracker.update(uid, Category, cat)
         results.append(result.contents())
     return results
 

sefaria/helper/topic.py

@@ -280,8 +280,12 @@ def curated_primacy(order_dict, lang):
     return (bord.get('numDatasource', 0) * bord.get('tfidf', 0)) - (aord.get('numDatasource', 0) * aord.get('tfidf', 0))
 
 
-def get_random_topic(good_to_promote=True) -> Optional[Topic]:
-    query = {"good_to_promote": True} if good_to_promote else {}
+def get_random_topic(pool=None) -> Optional[Topic]:
+    """
+    :param pool: name of the pool from which to select the topic. If `None`, all topics are considered.
+    :return: Returns a random topic from the database. If you provide `pool`, then the selection is limited to topics in that pool.
+    """
+    query = {"pools": pool} if pool else {}
     random_topic_dict = list(db.topics.aggregate([
         {"$match": query},
         {"$sample": {"size": 1}}
@@ -960,33 +964,18 @@ def calculate_popular_writings_for_authors(top_n, min_pr):
                 "order": {"custom_order": rd['pagesheetrank']}
             }).save()
 
-
 def recalculate_secondary_topic_data():
-    # run before everything else because this creates new links
-    calculate_popular_writings_for_authors(100, 300)
+    sheet_source_links = RefTopicLinkSet({'pools': 'textual'})
+    sheet_topic_links = RefTopicLinkSet({'pools': 'sheets'})
+    sheet_related_links = IntraTopicLinkSet()
 
-    sheet_source_links, sheet_related_links, sheet_topic_links = generate_all_topic_links_from_sheets()
     related_links = update_intra_topic_link_orders(sheet_related_links)
-    all_ref_links = update_ref_topic_link_orders(sheet_source_links, sheet_topic_links)
-
-    # now that we've gathered all the new links, delete old ones and insert new ones
-    RefTopicLinkSet({"generatedBy": TopicLinkHelper.generated_by_sheets}).delete()
-    RefTopicLinkSet({"is_sheet": True}).delete()
-    IntraTopicLinkSet({"generatedBy": TopicLinkHelper.generated_by_sheets}).delete()
-    print(f"Num Ref Links {len(all_ref_links)}")
-    print(f"Num Intra Links {len(related_links)}")
-    print(f"Num to Update {len(list(filter(lambda x: getattr(x, '_id', False), all_ref_links + related_links)))}")
-    print(f"Num to Insert {len(list(filter(lambda x: not getattr(x, '_id', False), all_ref_links + related_links)))}")
+    all_ref_links = update_ref_topic_link_orders(sheet_source_links.array(), sheet_topic_links.array())
 
     db.topic_links.bulk_write([
         UpdateOne({"_id": l._id}, {"$set": {"order": l.order}})
-        if getattr(l, "_id", False) else
-        InsertOne(l.contents(for_db=True))
         for l in (all_ref_links + related_links)
     ])
-    add_num_sources_to_topics()
-    make_titles_unique()
-
 
 def set_all_slugs_to_primary_title():
     # reset all slugs to their primary titles, if they have drifted away

sefaria/model/abstract.py

@@ -244,16 +244,12 @@ def _validate(self):
                              " not in " + ",".join(self.required_attrs) + " or " + ",".join(self.optional_attrs))
                 return False
         """
-        for attr, schema in self.attr_schemas.items():
-            v = Validator(schema)
-            try:
-                value = getattr(self, attr)
-                if not v.validate(value):
-                    raise InputError(v.errors)
-            except AttributeError:
-                # not checking here if value exists, that is done above.
-                # assumption is if value doesn't exist, it's optional
-                pass
+        schema = self.attr_schemas
+        for key in schema:
+            schema[key]['allow_unknown'] = schema[key].get('allow_unknown', False)  # allow unknowns only in the root
+        v = Validator(schema, allow_unknown=True)
+        if not v.validate(self._saveable_attrs()):
+            raise InputError(v.errors)
         return True
 
     def _normalize(self):

sefaria/model/lexicon.py

@@ -61,7 +61,8 @@ class Lexicon(abst.AbstractMongoRecord):
         'index_title',          # The title of the Index record that corresponds to this Lexicon
         'version_title',        # The title of the Version record that corresponds to this Lexicon
         'version_lang',         # The language of the Version record that corresponds to this Lexicon
-        'should_autocomplete'   # enables search box
+        'should_autocomplete',   # enables search box
+        'needsRefsWrapping'
     ]
 
     def word_count(self):
@@ -357,6 +358,23 @@ def as_strings(self, with_headword=True):
         return ['<br>'.join(strings)]
 
 
+class KovetzYesodotEntry(DictionaryEntry):
+    required_attrs = DictionaryEntry.required_attrs + ["content", "rid"]
+
+    def headword_string(self):
+        return f'<big><b>{self.headword}</b></big>'
+
+    def as_strings(self, with_headword=True):
+        strings = []
+        if with_headword:
+            strings.append(self.headword_string())
+        for key, value in self.content.items():
+            if key != 'reference':
+                strings.append(f'<br><small>{key}</small>')
+            strings += value
+        return ['<br>'.join(strings)]
+
+
 class LexiconEntrySubClassMapping(object):
     lexicon_class_map = {
         'BDB Augmented Strong': StrongsDictionaryEntry,
@@ -367,7 +385,8 @@ class LexiconEntrySubClassMapping(object):
         'Sefer HaShorashim': HebrewDictionaryEntry,
         'Animadversions by Elias Levita on Sefer HaShorashim': HebrewDictionaryEntry,
         'BDB Dictionary': BDBEntry,
-        'BDB Aramaic Dictionary': BDBEntry
+        'BDB Aramaic Dictionary': BDBEntry,
+        'Kovetz Yesodot VaChakirot': KovetzYesodotEntry,
     }
 
     @classmethod