Merge pull request #1610 from Sefaria/feature/sc-20203/internal-tooli…

…ng-for-learning-team

Feature/sc 20203/internal tooling for learning team

reader/views.py

@@ -62,7 +62,7 @@
 from sefaria.helper.topic import get_topic, get_all_topics, get_topics_for_ref, get_topics_for_book, \
                                 get_bulk_topics, recommend_topics, get_top_topic, get_random_topic, \
                                 get_random_topic_source, edit_topic_source, \
-                                update_order_of_topic_sources, delete_ref_topic_link
+                                update_order_of_topic_sources, delete_ref_topic_link, update_authors_place_and_time
 from sefaria.helper.community_page import get_community_page_items
 from sefaria.helper.file import get_resized_file
 from sefaria.image_generator import make_img_http_response
@@ -78,7 +78,7 @@
 from sefaria.utils.user import delete_user_account
 from django.core.mail import EmailMultiAlternatives
 from babel import Locale
-from sefaria.helper.topic import update_topic
+from sefaria.helper.topic import update_topic, update_topic_titles
 from sefaria.helper.category import update_order_of_category_children, check_term
 
 if USE_VARNISH:
@@ -1662,6 +1662,7 @@ def index_api(request, title, raw=False):
 
     if request.method == "POST":
         # use the update function if update is in the params
+
         func = tracker.update if request.GET.get("update", False) else tracker.add
         j = json.loads(request.POST.get("json"))
         if not j:
@@ -1694,6 +1695,7 @@ def protected_index_post(request):
             return jsonResponse(
                 func(request.user.id, Index, j, raw=raw).contents(raw=raw)
             )
+
         return protected_index_post(request)
 
     if request.method == "DELETE":
@@ -3100,17 +3102,19 @@ def add_new_topic_api(request):
         data = json.loads(request.POST["json"])
         isTopLevelDisplay = data["category"] == Topic.ROOT
         t = Topic({'slug': "", "isTopLevelDisplay": isTopLevelDisplay, "data_source": "sefaria", "numSources": 0})
-        t.add_title(data["title"], 'en', True, True)
-        if "heTitle" in data:
-            t.add_title(data["heTitle"], "he", True, True)
+        update_topic_titles(t, data)
 
         if not isTopLevelDisplay:  # not Top Level so create an IntraTopicLink to category
             new_link = IntraTopicLink({"toTopic": data["category"], "fromTopic": t.slug, "linkType": "displays-under", "dataSource": "sefaria"})
             new_link.save()
 
+        if data["category"] == 'authors':
+            t = update_authors_place_and_time(t, data)
+
         t.description_published = True
         t.data_source = "sefaria"  # any topic edited manually should display automatically in the TOC and this flag ensures this
-        t.change_description(data["description"], data.get("catDescription", None))
+        if "description" in data:
+            t.change_description(data["description"], data.get("categoryDescription", None))
         t.save()
 
         library.build_topic_auto_completer()
@@ -3163,8 +3167,6 @@ def topics_api(request, topic, v2=False):
         topic_obj = Topic().load({'slug': topic_data["origSlug"]})
         topic_data["manual"] = True
         author_status_changed = (topic_data["category"] == "authors") ^ (topic_data["origCategory"] == "authors")
-        if topic_data["category"] == topic_data["origCategory"]:
-            topic_data.pop("category")  # no need to check category in update_topic
         topic_obj = update_topic(topic_obj, **topic_data)
         if author_status_changed:
             library.build_topic_auto_completer()

requirements.txt

@@ -25,6 +25,7 @@ PyJWT==1.7.1 # pinned b/c current version 2.0.0 breaks simplejwt. waiting for 2.
 elasticsearch==7.9.1
 elasticsearch_dsl==7.3.0
 geojson==2.5.0
+geopy==2.3.0
 gevent==20.12.0
 google-api-python-client==1.12.5
 google-cloud-logging==1.15.1

sefaria/helper/descriptions.py

@@ -12,6 +12,38 @@
 from sefaria.system.exceptions import DuplicateRecordError
 from sefaria.model.abstract import SluggedAbstractMongoRecord
 
+def create_era_link(topic, prev_era_to_delete=None):
+    era_slug_map = {
+        "GN": "geon-person",
+        "RI": "rishon-person",
+        "AH": "achron-person",
+        "CO": "modern-person",
+        "KG": "mishnaic-people",
+        "PT": "mishnaic-people",
+        "T": "mishnaic-people",
+        "A": "talmudic-people",
+    }
+
+    isa_object_aggregate_map = {
+        'tosafot': 'group-of-rishon-people'
+    }
+
+    if prev_era_to_delete:
+        to_topic = isa_object_aggregate_map.get(topic.slug, era_slug_map[prev_era_to_delete])
+        prev_link = IntraTopicLink().load({'toTopic': to_topic, 'fromTopic': topic.slug, 'linkType': 'is-a'})
+        if prev_link:
+            prev_link.delete()
+
+    to_topic = isa_object_aggregate_map.get(topic.slug, era_slug_map[topic.get_property('era')])
+    itl = IntraTopicLink({
+        "toTopic": to_topic,
+        "fromTopic": topic.slug,
+        "linkType": "is-a",
+        "dataSource": "sefaria",
+        "generatedBy": "update_authors_data"
+    })
+    itl.save()
+
 
 
 def update_authors_data():
@@ -41,20 +73,6 @@ def update_authors_data():
         "Achronim": "AH",
         "Contemporary": "CO"
     }
-    era_slug_map = {
-        "GN": "geon-person",
-        "RI": "rishon-person",
-        "AH": "achron-person",
-        "CO": "modern-person",
-        "KG": "mishnaic-people",
-        "PT": "mishnaic-people",
-        "T": "mishnaic-people",
-        "A": "talmudic-people",
-    }
-
-    isa_object_aggregate_map = {
-        'tosafot': 'group-of-rishon-people'
-    }
 
     url = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSx60DLNs8Dp0l2xpsPjrxD3dBpIKASXSBiE-zjq74SvUIc-hD-mHwCxsuJpQYNVHIh7FDBwx7Pp9zR/pub?gid=0&single=true&output=csv'
     response = requests.get(url)
@@ -166,16 +184,8 @@ def _(p: Topic, attr, value):
             error_texts.append(str(e))
 
         if p.get_property('era'):
-            to_topic = isa_object_aggregate_map.get(p.slug, era_slug_map[p.get_property('era')])
-            itl = IntraTopicLink({
-                "toTopic": to_topic,
-                "fromTopic": p.slug,
-                "linkType": "is-a",
-                "dataSource": "sefaria",
-                "generatedBy": "update_authors_data"
-            })
             try:
-                itl.save()
+                create_era_link(p)
             except DuplicateRecordError as e:
                 error_texts.append(str(e))
 

sefaria/helper/topic.py

@@ -5,13 +5,15 @@
 from collections import defaultdict
 from functools import cmp_to_key
 from sefaria.model import *
+from sefaria.model.place import process_topic_place_change
 from sefaria.system.exceptions import InputError
 from sefaria.model.topic import TopicLinkHelper
 from sefaria.system.database import db
 from sefaria.system.cache import django_cache
 
 import structlog
 from sefaria import tracker
+from sefaria.helper.descriptions import create_era_link
 logger = structlog.get_logger(__name__)
 
 def get_topic(v2, topic, with_html=True, with_links=True, annotate_links=True, with_refs=True, group_related=True, annotate_time_period=False, ref_link_type_filters=None, with_indexes=True):
@@ -711,7 +713,7 @@ def get_sheet_order(topic_slug, sheet_id):
         else:
             sheet = db.sheets.find_one({"id": sheet_id}, {"views": 1, "includedRefs": 1, "dateCreated": 1, "options": 1, "title": 1, "topics": 1})
             includedRefs = []
-            for tref in sheet['includedRefs']:                
+            for tref in sheet['includedRefs']:
                 try:
                     oref = get_ref_safely(tref)
                     if oref is None:
@@ -911,7 +913,7 @@ def calculate_popular_writings_for_authors(top_n, min_pr):
             continue
         if getattr(oref.index, 'authors', None) is None: continue
         for author in oref.index.authors:
-            by_author[author] += [rd.contents()]    
+            by_author[author] += [rd.contents()]
     for author, rd_list in by_author.items():
         rd_list = list(filter(lambda x: x['pagesheetrank'] > min_pr, rd_list))
         if len(rd_list) == 0: continue
@@ -1037,27 +1039,64 @@ def topic_change_category(topic_obj, new_category, old_category="", rebuild=Fals
         rebuild_topic_toc(topic_obj, category_changed=True)
     return topic_obj
 
+def update_topic_titles(topic_obj, data):
+    for lang in ['en', 'he']:
+        for title in topic_obj.get_titles(lang):
+            topic_obj.remove_title(title, lang)
+        for title in data['altTitles'][lang]:
+            topic_obj.add_title(title, lang)
+
+    topic_obj.add_title(data['title'], 'en', True, True)
+    topic_obj.add_title(data['heTitle'], 'he', True, True)
+    return topic_obj
+
+
+def update_authors_place_and_time(topic_obj, data, dataSource='learning-team-editing-tool'):
+    # update place info added to author, then update year and era info
+    if not hasattr(topic_obj, 'properties'):
+        topic_obj.properties = {}
+    process_topic_place_change(topic_obj, data)
+    topic_obj = update_author_era(topic_obj, data, dataSource=dataSource)
+
+    return topic_obj
+
+def update_properties(topic_obj, dataSource, k, v):
+    if v == '':
+        topic_obj.properties.pop(k, None)
+    else:
+        topic_obj.properties[k] = {'value': v, 'dataSource': dataSource}
+
+def update_author_era(topic_obj, data, dataSource='learning-team-editing-tool'):
+    for k in ["birthYear", "deathYear"]:
+        if k in data.keys():    # only change property value if key is in data, otherwise it indicates no change
+            year = data[k]
+            update_properties(topic_obj, dataSource, k, year)
+
+    if 'era' in data.keys():    # only change property value if key is in data, otherwise it indicates no change
+        prev_era = topic_obj.properties.get('era', {}).get('value')
+        era = data['era']
+        update_properties(topic_obj, dataSource, 'era', era)
+        if era != '':
+            create_era_link(topic_obj, prev_era_to_delete=prev_era)
+    return topic_obj
 
 
 def update_topic(topic_obj, **kwargs):
     """
     Can update topic object's title, hebrew title, category, description, and categoryDescription fields
     :param topic_obj: (Topic) The topic to update
-    :param **kwargs can be title, heTitle, category, description, catDescription, and rebuild_toc where `title`, `heTitle`,
-         and `category` are strings. `description` and `catDescription` are dictionaries where the fields are `en` and `he`.
+    :param **kwargs can be title, heTitle, category, description, categoryDescription, and rebuild_toc where `title`, `heTitle`,
+         and `category` are strings. `description` and `categoryDescription` are dictionaries where the fields are `en` and `he`.
          The `category` parameter should be the slug of the new category. `rebuild_topic_toc` is a boolean and is assumed to be True
     :return: (model.Topic) The modified topic
     """
     old_category = ""
     orig_slug = topic_obj.slug
+    update_topic_titles(topic_obj, kwargs)
+    if kwargs.get('category') == 'authors':
+        topic_obj = update_authors_place_and_time(topic_obj, kwargs)
 
-    if 'title' in kwargs and kwargs['title'] != topic_obj.get_primary_title('en'):
-        topic_obj.add_title(kwargs['title'], 'en', True, True)
-
-    if 'heTitle' in kwargs and kwargs['heTitle'] != topic_obj.get_primary_title('he'):
-        topic_obj.add_title(kwargs['heTitle'], 'he', True, True)
-
-    if 'category' in kwargs:
+    if 'category' in kwargs and kwargs['category'] != kwargs['origCategory']:
         orig_link = IntraTopicLink().load({"linkType": "displays-under", "fromTopic": topic_obj.slug, "toTopic": {"$ne": topic_obj.slug}})
         old_category = orig_link.toTopic if orig_link else Topic.ROOT
         if old_category != kwargs['category']:
@@ -1067,8 +1106,8 @@ def update_topic(topic_obj, **kwargs):
         topic_obj.data_source = "sefaria"  # any topic edited manually should display automatically in the TOC and this flag ensures this
         topic_obj.description_published = True
 
-    if "description" in kwargs:
-        topic_obj.change_description(kwargs["description"], kwargs.get("catDescription", None))
+    if "description" in kwargs or "categoryDescription" in kwargs:
+        topic_obj.change_description(kwargs.get("description", None), kwargs.get("categoryDescription", None))
 
     topic_obj.save()
 
@@ -1211,7 +1250,7 @@ def delete_ref_topic_link(tref, to_topic, link_type, lang):
     if link is None:
         return {"error": f"Link between {tref} and {to_topic} doesn't exist."}
 
-    if lang in link.order['availableLangs']:   
+    if lang in link.order['availableLangs']:
         link.order['availableLangs'].remove(lang)
     if lang in link.order['curatedPrimacy']:
         link.order['curatedPrimacy'].pop(lang)

sefaria/model/dependencies.py

@@ -2,7 +2,7 @@
 dependencies.py -- list cross model dependencies and subscribe listeners to changes.
 """
 
-from . import abstract, link, note, history, schema, text, layer, version_state, timeperiod, garden, notification, collection, library, category, ref_data, user_profile, manuscript, topic
+from . import abstract, link, note, history, schema, text, layer, version_state, timeperiod, garden, notification, collection, library, category, ref_data, user_profile, manuscript, topic, place
 
 from .abstract import subscribe, cascade, cascade_to_list, cascade_delete, cascade_delete_to_list
 import sefaria.system.cache as scache
@@ -11,7 +11,8 @@
 subscribe(text.process_index_change_in_core_cache,                      text.Index, "save")
 subscribe(version_state.create_version_state_on_index_creation,         text.Index, "save")
 subscribe(text.process_index_change_in_toc,                             text.Index, "save")
-
+subscribe(place.process_index_place_change, text.Index, 'attributeChange', 'compPlace')
+subscribe(place.process_index_place_change, text.Index, 'attributeChange', 'pubPlace')
 
 # Index Name Change
 subscribe(text.process_index_title_change_in_core_cache,                text.Index, "attributeChange", "title")
@@ -75,6 +76,7 @@ def process_version_title_change_in_search(ver, **kwargs):
 subscribe(topic.process_topic_description_change,                       topic.Topic, "attributeChange", "description")
 subscribe(topic.process_topic_delete,                                 topic.AuthorTopic, "delete")
 
+
 # Terms
 # TODO cascade change to Term.name.
 # TODO Current locations where we know terms are used [Index, Categories]

sefaria/model/place.py

@@ -3,8 +3,9 @@
 from . import abstract as abst
 from . import schema
 from sefaria.system.exceptions import InputError
-
 import structlog
+from geopy.geocoders import Nominatim
+from sefaria.utils.hebrew import get_he_key
 logger = structlog.get_logger(__name__)
 
 class Place(abst.AbstractMongoRecord):
@@ -46,8 +47,24 @@ def primary_name(self, lang=None):
 
     def secondary_names(self, lang=None):
         return self.name_group.secondary_titles(lang)
-
-    ###
+
+    @classmethod 
+    def create_new_place(cls, en, he=None):
+        p = cls().load({'key': en})
+        if p:
+            return p
+        p = cls({'key': en})
+        p.name_group.add_title(en, 'en', True, True)
+        if he:
+            p.name_group.add_title(he, 'he', True, True)
+        p.city_to_coordinates(en)
+        p.save()
+        return p
+
+    def city_to_coordinates(self, city):
+        geolocator = Nominatim(user_agent='[email protected]')
+        location = geolocator.geocode(city)
+        self.point_location(lon=location.longitude, lat=location.latitude)
 
     def point_location(self, lon=None, lat=None):
         if lat is None and lon is None:
@@ -80,3 +97,24 @@ def asGeoJson(self, with_polygons=False, as_string=False):
             return geojson.dumps(geojson.FeatureCollection(features))
         else:
             return geojson.FeatureCollection(features)
+
+def process_index_place_change(indx, **kwargs):
+    key = kwargs['attr']
+    he_key = get_he_key(key)
+    he_new_val = getattr(indx, he_key, '')
+    if kwargs['new'] != '':
+        Place.create_new_place(en=kwargs['new'], he=he_new_val)
+
+def process_topic_place_change(topic_obj, data):
+    keys = ["birthPlace", "deathPlace"]
+    for key in keys:
+        if key in data.keys():  # only change property value if key is in data, otherwise it indicates no change
+            new_val = data[key]
+            if new_val != '':
+                he_key = get_he_key(key)
+                he_new_val = data.get(he_key, '')
+                place = Place.create_new_place(en=new_val, he=he_new_val)
+                topic_obj.properties[key] = {"value": place.primary_name('en'), 'dataSource': 'learning-team-editing-tool'}
+            else:
+                topic_obj.properties.pop(key, None)
+