From de6f4721978926aa5a139b3890173a122207ba23 Mon Sep 17 00:00:00 2001 From: saengel Date: Wed, 10 Jul 2024 16:02:19 +0300 Subject: [PATCH 001/151] fix(api): refine v1 texts error --- sefaria/model/text.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sefaria/model/text.py b/sefaria/model/text.py index 18a1e1a28f..5d99eae01d 100644 --- a/sefaria/model/text.py +++ b/sefaria/model/text.py @@ -3990,8 +3990,10 @@ def padded_ref(self): except AttributeError: # This is a schema node, try to get a default child if self.has_default_child(): return self.default_child_ref().padded_ref() + elif self.is_book_level(): + raise InputError("Please pass a more specific ref for this book, and try again. The ref you passed is a 'complex' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. ") else: - raise InputError("Can not pad a schema node ref") + raise InputError("Cannot pad a schema node ref.") d = self._core_dict() if self.is_talmud(): From c9d05f18517966244506219f81b53e2690f196af Mon Sep 17 00:00:00 2001 From: saengel Date: Sun, 14 Jul 2024 13:55:32 +0300 Subject: [PATCH 002/151] fix(api): First pass at removing 500 from v3 --- api/views.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index bd1525390a..f5aaa088c6 100644 --- a/api/views.py +++ b/api/views.py @@ -1,10 +1,12 @@ from sefaria.model import * from sefaria.model.text_reuqest_adapter import TextRequestAdapter from sefaria.client.util import jsonResponse +from sefaria.system.exceptions import InputError from django.views import View from .api_warnings import * + class Text(View): RETURN_FORMATS = ['default', 'wrap_all_entities', 'text_only', 'strip_only_footnotes'] @@ -53,6 +55,14 @@ def get(self, request, *args, **kwargs): if return_format not in self.RETURN_FORMATS: return jsonResponse({'error': f'return_format should be one of those formats: {self.RETURN_FORMATS}.'}, status=400) text_manager = TextRequestAdapter(self.oref, versions_params, fill_in_missing_segments, return_format) - data = text_manager.get_versions_for_query() - data = self._handle_warnings(data) - return jsonResponse(data) + + try: + # For a SchemaNode, data is an error which handle_warnings doesn't handle (or even get triggered?) + # How to trigger a 400 appropriate error with an appropriate message? + data = text_manager.get_versions_for_query() + data = self._handle_warnings(data) + return jsonResponse(data) + except InputError as e: + raise InputError(e) + + From 33d8b0d1a4a605b606c7ff6461759b3f6d7a3497 Mon Sep 17 00:00:00 2001 From: saengel Date: Mon, 15 Jul 2024 10:35:14 +0300 Subject: [PATCH 003/151] feat(api): scaffolding out 400 error instead of 500 --- api/views.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/api/views.py b/api/views.py index f5aaa088c6..14a4c2d3f5 100644 --- a/api/views.py +++ b/api/views.py @@ -56,13 +56,18 @@ def get(self, request, *args, **kwargs): return jsonResponse({'error': f'return_format should be one of those formats: {self.RETURN_FORMATS}.'}, status=400) text_manager = TextRequestAdapter(self.oref, versions_params, fill_in_missing_segments, return_format) + try: - # For a SchemaNode, data is an error which handle_warnings doesn't handle (or even get triggered?) - # How to trigger a 400 appropriate error with an appropriate message? + # Todo - Maybe this error should be inside the get_versions_for_query() fxn? data = text_manager.get_versions_for_query() - data = self._handle_warnings(data) - return jsonResponse(data) - except InputError as e: - raise InputError(e) + except Exception as e: + # Todo - which 400 code exactly to pass? Will have to check the message of the error to + # make sure you're sending the right response to the user. + return jsonResponse({'error': "Please pass a more specific Ref"}, status=400) + + data = self._handle_warnings(data) + return jsonResponse(data) + + From fcfbe65101d6dfa9d8abbd8beb275bb13d4a4dcb Mon Sep 17 00:00:00 2001 From: saengel Date: Mon, 15 Jul 2024 10:35:55 +0300 Subject: [PATCH 004/151] fix(api): update error message to include a link --- api/views.py | 2 +- sefaria/model/text.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 14a4c2d3f5..a73f33583d 100644 --- a/api/views.py +++ b/api/views.py @@ -62,7 +62,7 @@ def get(self, request, *args, **kwargs): data = text_manager.get_versions_for_query() except Exception as e: # Todo - which 400 code exactly to pass? Will have to check the message of the error to - # make sure you're sending the right response to the user. + # make sure you're sending the right response to the user. return jsonResponse({'error': "Please pass a more specific Ref"}, status=400) data = self._handle_warnings(data) diff --git a/sefaria/model/text.py b/sefaria/model/text.py index 5d99eae01d..7a82d742c7 100644 --- a/sefaria/model/text.py +++ b/sefaria/model/text.py @@ -3991,7 +3991,7 @@ def padded_ref(self): if self.has_default_child(): return self.default_child_ref().padded_ref() elif self.is_book_level(): - raise InputError("Please pass a more specific ref for this book, and try again. The ref you passed is a 'complex' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. ") + raise InputError("Please pass a more specific ref for this book, and try again. The ref you passed is a 'complex' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text") else: raise InputError("Cannot pad a schema node ref.") From f76702deee05ed294047046c8e6da7f685be11e8 Mon Sep 17 00:00:00 2001 From: saengel Date: Wed, 17 Jul 2024 13:56:28 +0300 Subject: [PATCH 005/151] feat(api): add message check --- api/views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/views.py b/api/views.py index a73f33583d..c4f7101388 100644 --- a/api/views.py +++ b/api/views.py @@ -58,13 +58,11 @@ def get(self, request, *args, **kwargs): try: - # Todo - Maybe this error should be inside the get_versions_for_query() fxn? data = text_manager.get_versions_for_query() - except Exception as e: - # Todo - which 400 code exactly to pass? Will have to check the message of the error to - # make sure you're sending the right response to the user. - return jsonResponse({'error': "Please pass a more specific Ref"}, status=400) - + except InputError as e: + # Todo - which 400 code exactly to pass? + if str(e) == "Can not get TextRange at this level, please provide a more precise reference": + return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) data = self._handle_warnings(data) return jsonResponse(data) From f87734dba5e992b7d6ca90c9e1b8b9b555c839b9 Mon Sep 17 00:00:00 2001 From: saengel Date: Wed, 17 Jul 2024 20:50:23 +0300 Subject: [PATCH 006/151] chore(api): remove todos --- api/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/views.py b/api/views.py index c4f7101388..a8d60b4662 100644 --- a/api/views.py +++ b/api/views.py @@ -60,7 +60,6 @@ def get(self, request, *args, **kwargs): try: data = text_manager.get_versions_for_query() except InputError as e: - # Todo - which 400 code exactly to pass? if str(e) == "Can not get TextRange at this level, please provide a more precise reference": return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) data = self._handle_warnings(data) From 6536f7459f32ca4759094c5a85903ebc4ed86242 Mon Sep 17 00:00:00 2001 From: saengel Date: Fri, 19 Jul 2024 10:05:35 +0300 Subject: [PATCH 007/151] chore(api): ensure model level error message is informative, and appropriate for the model --- sefaria/model/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/model/text.py b/sefaria/model/text.py index 7a82d742c7..f9bea61dae 100644 --- a/sefaria/model/text.py +++ b/sefaria/model/text.py @@ -3991,7 +3991,7 @@ def padded_ref(self): if self.has_default_child(): return self.default_child_ref().padded_ref() elif self.is_book_level(): - raise InputError("Please pass a more specific ref for this book, and try again. The ref you passed is a 'complex' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text") + raise InputError("Can not get TextRange at this level, please provide a more precise reference") else: raise InputError("Cannot pad a schema node ref.") From 390204e1cb03733c04f6b6254879ce2382998318 Mon Sep 17 00:00:00 2001 From: saengel Date: Fri, 19 Jul 2024 10:05:59 +0300 Subject: [PATCH 008/151] feat(api): add a v1 view error message including a link --- sefaria/system/decorators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sefaria/system/decorators.py b/sefaria/system/decorators.py index 94db751aea..73c1c74660 100644 --- a/sefaria/system/decorators.py +++ b/sefaria/system/decorators.py @@ -42,7 +42,10 @@ def wrapper(*args, **kwargs): except exps.InputError as e: logger.warning("An exception occurred processing request for '{}' while running {}. Caught as JSON".format(args[0].path, func.__name__), exc_info=True) request = args[0] - return jsonResponse({"error": str(e)}, callback=request.GET.get("callback", None)) + if str(e) == "Can not get TextRange at this level, please provide a more precise reference": + return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) + else: + return jsonResponse({"error": str(e)}, callback=request.GET.get("callback", None)) return result return wrapper From 2509488bf619aea2eac63ed2b067d0c38946bde4 Mon Sep 17 00:00:00 2001 From: saengel Date: Thu, 1 Aug 2024 13:11:00 -0400 Subject: [PATCH 009/151] fix(api): Move errors to view level --- reader/views.py | 8 ++++++-- sefaria/system/decorators.py | 5 +---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/reader/views.py b/reader/views.py index ef8682dd38..f1df2c94ee 100644 --- a/reader/views.py +++ b/reader/views.py @@ -1447,8 +1447,12 @@ def _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=comment return text if not multiple or abs(multiple) == 1: - text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad, - alts=alts, wrapLinks=wrapLinks, layer_name=layer_name) + try: + text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad, + alts=alts, wrapLinks=wrapLinks, layer_name=layer_name) + except Exception as e: + if str(e) == "Can not get TextRange at this level, please provide a more precise reference": + return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) return jsonResponse(text, cb) else: # Return list of many sections diff --git a/sefaria/system/decorators.py b/sefaria/system/decorators.py index 73c1c74660..94db751aea 100644 --- a/sefaria/system/decorators.py +++ b/sefaria/system/decorators.py @@ -42,10 +42,7 @@ def wrapper(*args, **kwargs): except exps.InputError as e: logger.warning("An exception occurred processing request for '{}' while running {}. Caught as JSON".format(args[0].path, func.__name__), exc_info=True) request = args[0] - if str(e) == "Can not get TextRange at this level, please provide a more precise reference": - return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) - else: - return jsonResponse({"error": str(e)}, callback=request.GET.get("callback", None)) + return jsonResponse({"error": str(e)}, callback=request.GET.get("callback", None)) return result return wrapper From ef5752b838f0ffecc82fc73a2d71c4ae4dba01e0 Mon Sep 17 00:00:00 2001 From: stevekaplan123 Date: Mon, 14 Oct 2024 10:01:42 +0300 Subject: [PATCH 010/151] chore: only show add to sheet if logged in --- static/js/ConnectionsPanel.jsx | 2 +- static/js/TextList.jsx | 2 +- static/js/TranslationsBox.jsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/static/js/ConnectionsPanel.jsx b/static/js/ConnectionsPanel.jsx index c040af9889..133e46e1e7 100644 --- a/static/js/ConnectionsPanel.jsx +++ b/static/js/ConnectionsPanel.jsx @@ -756,7 +756,7 @@ const ToolsList = ({ setConnectionsMode, toggleSignUpModal, openComparePanel, co // A list of Resources in addition to connection return (
- !Sefaria._uid ? toggleSignUpModal(SignUpModalKind.AddToSheet) : setConnectionsMode("Add To Sheet", { "addSource": "mainPanel" })} /> + {Sefaria._uid && setConnectionsMode("Add To Sheet", { "addSource": "mainPanel" })} />} setConnectionsMode("Lexicon")} /> {openComparePanel ? : null} !Sefaria._uid ? toggleSignUpModal(SignUpModalKind.Notes) : setConnectionsMode("Notes")} /> diff --git a/static/js/TextList.jsx b/static/js/TextList.jsx index 41c6efe5b8..faf4d09e6d 100644 --- a/static/js/TextList.jsx +++ b/static/js/TextList.jsx @@ -235,7 +235,7 @@ class TextList extends Component { /> - + {Sefaria._uid && } {Sefaria.is_moderator ? : null } diff --git a/static/js/TranslationsBox.jsx b/static/js/TranslationsBox.jsx index 8a1d7e9723..a8430fdc78 100644 --- a/static/js/TranslationsBox.jsx +++ b/static/js/TranslationsBox.jsx @@ -190,7 +190,7 @@ class VersionsTextList extends Component { /> - + {Sefaria._uid && }
); } From 353e652e87a5101e122fd9329d394e2d16983dda Mon Sep 17 00:00:00 2001 From: saengel Date: Sun, 10 Nov 2024 13:28:37 +0200 Subject: [PATCH 011/151] chore(api): Correct param type to enable accurate results --- sefaria/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/views.py b/sefaria/views.py index 80f3939e0d..8309b8d0ef 100644 --- a/sefaria/views.py +++ b/sefaria/views.py @@ -470,7 +470,7 @@ def bulktext_api(request, refs): g = lambda x: request.GET.get(x, None) min_char = int(g("minChar")) if g("minChar") else None max_char = int(g("maxChar")) if g("maxChar") else None - res = bundle_many_texts(refs, g("useTextFamily"), g("asSizedString"), min_char, max_char, g("transLangPref"), g("ven"), g("vhe")) + res = bundle_many_texts(refs, int(g("useTextFamily")), g("asSizedString"), min_char, max_char, g("transLangPref"), g("ven"), g("vhe")) resp = jsonResponse(res, cb) return resp From 25fc731124cdde62b0ab1d670bbb877fe6940797 Mon Sep 17 00:00:00 2001 From: saengel Date: Mon, 11 Nov 2024 15:14:53 +0200 Subject: [PATCH 012/151] chore(api errors): Add exception as an official exception vs string comp --- reader/views.py | 6 +++--- sefaria/system/exceptions.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/reader/views.py b/reader/views.py index f1df2c94ee..527a185afb 100644 --- a/reader/views.py +++ b/reader/views.py @@ -53,7 +53,7 @@ from sefaria.site.site_settings import SITE_SETTINGS from sefaria.system.multiserver.coordinator import server_coordinator from sefaria.system.decorators import catch_error_as_json, sanitize_get_params, json_response_decorator -from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError +from sefaria.system.exceptions import InputError, PartialRefInputError, BookNameError, NoVersionFoundError, DictionaryEntryNotFoundError, ComplexBookLevelRefError from sefaria.system.cache import django_cache from sefaria.system.database import db from sefaria.helper.search import get_query_obj @@ -1451,8 +1451,8 @@ def _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=comment text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad, alts=alts, wrapLinks=wrapLinks, layer_name=layer_name) except Exception as e: - if str(e) == "Can not get TextRange at this level, please provide a more precise reference": - return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) + if isinstance(e, ComplexBookLevelRefError): + return jsonResponse({'error': e.message}, status=400) return jsonResponse(text, cb) else: # Return list of many sections diff --git a/sefaria/system/exceptions.py b/sefaria/system/exceptions.py index 72d3493f78..d0029d2105 100644 --- a/sefaria/system/exceptions.py +++ b/sefaria/system/exceptions.py @@ -99,3 +99,9 @@ def __init__(self, method): self.method = method self.message = f"'{method}' is not a valid HTTP API method." super().__init__(self.message) + +class ComplexBookLevelRefError(Exception): + def __init__(self, book_ref): + self.book_ref = book_ref + self.message = f"Please pass a more specific ref for this book, and try again. You passed in a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text" + super().__init__(self.message) From 6bea38ee9b70c7f6905b831c805d8f79b57ebb40 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Mon, 11 Nov 2024 22:11:30 +0200 Subject: [PATCH 013/151] feat(topics): add topic_pool_link model --- admin_tools/__init__.py | 0 admin_tools/migrations/0001_initial.py | 24 ++++++++++++++++++ .../migrations/0002_delete_topicpoollink.py | 18 +++++++++++++ admin_tools/migrations/0003_topicpoollink.py | 25 +++++++++++++++++++ admin_tools/migrations/__init__.py | 0 admin_tools/models/__init__.py | 1 + admin_tools/models/topic_pool_link.py | 25 +++++++++++++++++++ sefaria/settings.py | 1 + 8 files changed, 94 insertions(+) create mode 100644 admin_tools/__init__.py create mode 100644 admin_tools/migrations/0001_initial.py create mode 100644 admin_tools/migrations/0002_delete_topicpoollink.py create mode 100644 admin_tools/migrations/0003_topicpoollink.py create mode 100644 admin_tools/migrations/__init__.py create mode 100644 admin_tools/models/__init__.py create mode 100644 admin_tools/models/topic_pool_link.py diff --git a/admin_tools/__init__.py b/admin_tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/admin_tools/migrations/0001_initial.py b/admin_tools/migrations/0001_initial.py new file mode 100644 index 0000000000..ec43fcb95a --- /dev/null +++ b/admin_tools/migrations/0001_initial.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-11 17:45 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TopicPoolLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool', models.CharField(max_length=255)), + ('topic_slug', models.CharField(max_length=255)), + ], + ), + ] diff --git a/admin_tools/migrations/0002_delete_topicpoollink.py b/admin_tools/migrations/0002_delete_topicpoollink.py new file mode 100644 index 0000000000..98e95d6eef --- /dev/null +++ b/admin_tools/migrations/0002_delete_topicpoollink.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-11 18:42 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('admin_tools', '0001_initial'), + ] + + operations = [ + migrations.DeleteModel( + name='TopicPoolLink', + ), + ] diff --git a/admin_tools/migrations/0003_topicpoollink.py b/admin_tools/migrations/0003_topicpoollink.py new file mode 100644 index 0000000000..95558d20a4 --- /dev/null +++ b/admin_tools/migrations/0003_topicpoollink.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-11 18:43 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('admin_tools', '0002_delete_topicpoollink'), + ] + + operations = [ + migrations.CreateModel( + name='TopicPoolLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool', models.CharField(max_length=255)), + ('topic_slug', models.CharField(max_length=255)), + ], + ), + ] diff --git a/admin_tools/migrations/__init__.py b/admin_tools/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/admin_tools/models/__init__.py b/admin_tools/models/__init__.py new file mode 100644 index 0000000000..6eaf38f2f7 --- /dev/null +++ b/admin_tools/models/__init__.py @@ -0,0 +1 @@ +from .topic_pool_link import TopicPoolLink diff --git a/admin_tools/models/topic_pool_link.py b/admin_tools/models/topic_pool_link.py new file mode 100644 index 0000000000..17a64a726b --- /dev/null +++ b/admin_tools/models/topic_pool_link.py @@ -0,0 +1,25 @@ +from django.db import models + + +class TopicPoolLinkManager(models.Manager): + def get_random_topic_slugs(self, pool=None, limit=10) -> list[str]: + query_set = self.get_queryset() + if pool: + query_set = query_set.filter(pool=pool) + query_set = query_set.values('topic_slug').distinct().order_by('?')[:limit] + return [x['topic_slug'] for x in query_set] + + +class TopicPoolLink(models.Model): + pool = models.CharField(max_length=255) + topic_slug = models.CharField(max_length=255) + objects = TopicPoolLinkManager() + + def __str__(self): + return f"{self.pool} <> {self.topic_slug}" + + + + + + diff --git a/sefaria/settings.py b/sefaria/settings.py index 7eea25ff70..1a425e5392 100644 --- a/sefaria/settings.py +++ b/sefaria/settings.py @@ -144,6 +144,7 @@ 'reader', 'sourcesheets', 'sefaria.gauth', + 'admin_tools', 'captcha', 'django.contrib.admin', 'anymail', From ac1fa70e41a40603601056722936ba45bebc9f22 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Mon, 11 Nov 2024 22:11:50 +0200 Subject: [PATCH 014/151] refactor(topics): modify random topic api to use topic pool link model --- sefaria/helper/topic.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index 8203a06b31..d69e3aa0cd 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -285,15 +285,12 @@ def get_random_topic(pool=None) -> Optional[Topic]: :param pool: name of the pool from which to select the topic. If `None`, all topics are considered. :return: Returns a random topic from the database. If you provide `pool`, then the selection is limited to topics in that pool. """ - query = {"pools": pool} if pool else {} - random_topic_dict = list(db.topics.aggregate([ - {"$match": query}, - {"$sample": {"size": 1}} - ])) - if len(random_topic_dict) == 0: + from admin_tools.models import TopicPoolLink + random_topic_slugs = TopicPoolLink.objects.get_random_topic_slugs(pool=pool, limit=1) + if len(random_topic_slugs) == 0: return None - return Topic(random_topic_dict[0]) + return Topic.init(random_topic_slugs[0]) def get_random_topic_source(topic:Topic) -> Optional[Ref]: From 110b05104043f56aed6808fa097105eca930917a Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Mon, 11 Nov 2024 22:19:00 +0200 Subject: [PATCH 015/151] refactor(topics): change pool to 'promoted' --- reader/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reader/views.py b/reader/views.py index aafc905ef2..35f191eea4 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4230,7 +4230,7 @@ def random_by_topic_api(request): Returns Texts API data for a random text taken from popular topic tags """ cb = request.GET.get("callback", None) - random_topic = get_random_topic('torahtab') + random_topic = get_random_topic('promoted') if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) From cce7daf4c4a59fec42bc1386d42f993ee706209b Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Tue, 12 Nov 2024 13:04:31 +0200 Subject: [PATCH 016/151] refactor(topics): move management of pools to use TopicLinkPool model --- admin_tools/models/topic_pool_link.py | 15 ++++++++++ reader/views.py | 3 +- sefaria/model/topic.py | 43 ++++++++++----------------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/admin_tools/models/topic_pool_link.py b/admin_tools/models/topic_pool_link.py index 17a64a726b..f7426cf872 100644 --- a/admin_tools/models/topic_pool_link.py +++ b/admin_tools/models/topic_pool_link.py @@ -1,4 +1,11 @@ from django.db import models +from enum import Enum + + +class PoolType(Enum): + TEXTUAL = "textual" + SHEETS = "sheets" + PROMOTED = "promoted" class TopicPoolLinkManager(models.Manager): @@ -9,12 +16,20 @@ def get_random_topic_slugs(self, pool=None, limit=10) -> list[str]: query_set = query_set.values('topic_slug').distinct().order_by('?')[:limit] return [x['topic_slug'] for x in query_set] + @staticmethod + def get_pools_by_topic_slug(topic_slug) -> list[str]: + query_set = TopicPoolLink.objects.filter(topic_slug=topic_slug).values('pool').distinct() + return [x['pool'] for x in query_set] + class TopicPoolLink(models.Model): pool = models.CharField(max_length=255) topic_slug = models.CharField(max_length=255) objects = TopicPoolLinkManager() + class Meta: + unique_together = ('pool', 'topic_slug') + def __str__(self): return f"{self.pool} <> {self.topic_slug}" diff --git a/reader/views.py b/reader/views.py index 35f191eea4..1395cf1ac2 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4229,8 +4229,9 @@ def random_by_topic_api(request): """ Returns Texts API data for a random text taken from popular topic tags """ + from admin_tools.models.topic_pool_link import PoolType cb = request.GET.get("callback", None) - random_topic = get_random_topic('promoted') + random_topic = get_random_topic(PoolType.PROMOTED.value) if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index ef48716027..c126c8b2d0 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -1,9 +1,11 @@ from enum import Enum from typing import Union, Optional +from django.db.utils import IntegrityError from . import abstract as abst from .schema import AbstractTitledObject, TitleGroup from .text import Ref, IndexSet, AbstractTextRecord, Index, Term from .category import Category +from admin_tools.models.topic_pool_link import TopicPoolLink, PoolType from sefaria.system.exceptions import InputError, DuplicateRecordError from sefaria.model.timeperiod import TimePeriod, LifePeriod from sefaria.system.validators import validate_url @@ -121,11 +123,6 @@ def __hash__(self): return hash((self.collective_title, self.base_cat_path)) -class Pool(Enum): - TEXTUAL = "textual" - SHEETS = "sheets" - - class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): collection = 'topics' history_noun = 'topic' @@ -163,8 +160,6 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'pools', # list of strings, any of them represents a pool that this topic is member of ] - allowed_pools = [pool.value for pool in Pool] + ['torahtab'] - attr_schemas = { "image": { 'type': 'dict', @@ -176,14 +171,7 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): 'schema': {'en': {'type': 'string', 'required': True}, 'he': {'type': 'string', 'required': True}}}} }, - 'pools': { - 'type': 'list', - 'schema': { - 'type': 'string', - 'allowed': allowed_pools - } - } - } + } ROOT = "Main Menu" # the root of topic TOC is not a topic, so this is a fake slug. we know it's fake because it's not in normal form # this constant is helpful in the topic editor tool functions in this file @@ -200,6 +188,7 @@ def load(self, query, proj=None): def _set_derived_attributes(self): self.set_titles(getattr(self, "titles", None)) + self.pools = TopicPoolLink.objects.get_pools_by_topic_slug(getattr(self, "slug", None)) if self.__class__ != Topic and not getattr(self, "subclass", False): # in a subclass. set appropriate "subclass" attribute setattr(self, "subclass", self.reverse_subclass_map[self.__class__.__name__]) @@ -224,10 +213,6 @@ def _normalize(self): displays_under_link = IntraTopicLink().load({"fromTopic": slug, "linkType": "displays-under"}) if getattr(displays_under_link, "toTopic", "") == "authors": self.subclass = "author" - if self.get_pools(): - self.pools = sorted(set(self.get_pools())) - elif hasattr(self, 'pools'): - delattr(self, 'pools') def _sanitize(self): super()._sanitize() @@ -237,17 +222,23 @@ def _sanitize(self): p[k] = bleach.clean(v, tags=[], strip=True) setattr(self, attr, p) - def get_pools(self): + def get_pools(self) -> list[str]: return getattr(self, 'pools', []) - def has_pool(self, pool): + def has_pool(self, pool: str) -> bool: return pool in self.get_pools() - def add_pool(self, pool): #does not save! + def add_pool(self, pool: str) -> None: + try: + link = TopicPoolLink(pool=pool, topic_slug=self.slug) + link.save() + except IntegrityError: + raise DuplicateRecordError(f"'{pool}'<>'{self.slug}' link already exists in TopicPoolLink table.") self.pools = self.get_pools() self.pools.append(pool) - def remove_pool(self, pool): #does not save! + def remove_pool(self, pool) -> None: + TopicPoolLink.objects.filter(pool=pool, topic_slug=self.slug).delete() pools = self.get_pools() pools.remove(pool) @@ -498,8 +489,6 @@ def get_ref_links(self, is_sheet, query_kwargs=None, **kwargs): def contents(self, **kwargs): mini = kwargs.get('minify', False) d = {'slug': self.slug} if mini else super(Topic, self).contents(**kwargs) - if kwargs.get('remove_pools', True): - d.pop('pools', None) d['primaryTitle'] = {} for lang in ('en', 'he'): d['primaryTitle'][lang] = self.get_primary_title(lang=lang, with_disambiguation=kwargs.get('with_disambiguation', True)) @@ -565,7 +554,7 @@ def update_after_link_change(self, pool): updating the pools 'sheets' or 'textual' according to the existence of links and the numSources :param pool: 'sheets' or 'textual' """ - links = self.get_ref_links(pool == Pool.SHEETS.value) + links = self.get_ref_links(pool == PoolType.SHEETS.value) if self.has_pool(pool) and not links: self.remove_pool(pool) elif not self.has_pool(pool) and links: @@ -970,7 +959,7 @@ def set_description(self, lang, title, prompt): return self def get_related_pool(self): - return Pool.SHEETS.value if self.is_sheet else Pool.TEXTUAL.value + return PoolType.SHEETS.value if self.is_sheet else PoolType.TEXTUAL.value def get_topic(self): return Topic().load({'slug': self.toTopic}) From 94dee446314d355ed6adabba8259b1b1495ca6a5 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Tue, 12 Nov 2024 13:04:51 +0200 Subject: [PATCH 017/151] chore(topics): add uniqueness constraint on topicpoollink --- .../migrations/0004_auto_20241111_2328.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 admin_tools/migrations/0004_auto_20241111_2328.py diff --git a/admin_tools/migrations/0004_auto_20241111_2328.py b/admin_tools/migrations/0004_auto_20241111_2328.py new file mode 100644 index 0000000000..866e648b61 --- /dev/null +++ b/admin_tools/migrations/0004_auto_20241111_2328.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-12 03:28 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('admin_tools', '0003_topicpoollink'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='topicpoollink', + unique_together=set([('pool', 'topic_slug')]), + ), + ] From 129529bf456f3fb5c35cf58db31410b939b4c94c Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Tue, 12 Nov 2024 13:31:52 +0200 Subject: [PATCH 018/151] chore(topics): add migrate_good_to_promote_to_topic_pools.py --- .../migrate_good_to_promote_to_topic_pools.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scripts/migrations/migrate_good_to_promote_to_topic_pools.py diff --git a/scripts/migrations/migrate_good_to_promote_to_topic_pools.py b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py new file mode 100644 index 0000000000..1259dfa4df --- /dev/null +++ b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py @@ -0,0 +1,15 @@ +import django +django.setup() +from sefaria.model import * +from admin_tools.models.topic_pool_link import PoolType, TopicPoolLink + + +def run(): + ts = TopicSet({'good_to_promote': True}) + for topic in ts: + link = TopicPoolLink(topic_slug=topic.slug, pool=PoolType.PROMOTED.value) + link.save() + + +if __name__ == "__main__": + run() From 599180dbdd94ce6a9b6d013b82aac6d91641767b Mon Sep 17 00:00:00 2001 From: saengel Date: Wed, 13 Nov 2024 12:43:33 +0200 Subject: [PATCH 019/151] chore(api errors): Raise exception, find the right place in model for exception --- api/views.py | 14 +++++++++----- reader/views.py | 2 +- sefaria/model/text.py | 8 ++++---- sefaria/system/exceptions.py | 8 ++++++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/api/views.py b/api/views.py index a8d60b4662..b91fa760c4 100644 --- a/api/views.py +++ b/api/views.py @@ -1,7 +1,7 @@ from sefaria.model import * from sefaria.model.text_reuqest_adapter import TextRequestAdapter from sefaria.client.util import jsonResponse -from sefaria.system.exceptions import InputError +from sefaria.system.exceptions import InputError, ComplexBookLevelRefError from django.views import View from .api_warnings import * @@ -59,12 +59,16 @@ def get(self, request, *args, **kwargs): try: data = text_manager.get_versions_for_query() - except InputError as e: - if str(e) == "Can not get TextRange at this level, please provide a more precise reference": - return jsonResponse({'error': "Please pass a more specific ref for this book, and try again. The ref you passed is a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text"}, status=400) - data = self._handle_warnings(data) + data = self._handle_warnings(data) + + except Exception as e: + if isinstance(e, InputError): + return jsonResponse({'error': e.message}) + return jsonResponse(data) + + diff --git a/reader/views.py b/reader/views.py index 527a185afb..f3fae99373 100644 --- a/reader/views.py +++ b/reader/views.py @@ -1450,7 +1450,7 @@ def _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=comment try: text = _get_text(oref, versionEn=versionEn, versionHe=versionHe, commentary=commentary, context=context, pad=pad, alts=alts, wrapLinks=wrapLinks, layer_name=layer_name) - except Exception as e: + except InputError as e: if isinstance(e, ComplexBookLevelRefError): return jsonResponse({'error': e.message}, status=400) return jsonResponse(text, cb) diff --git a/sefaria/model/text.py b/sefaria/model/text.py index f9bea61dae..d9c78cc652 100644 --- a/sefaria/model/text.py +++ b/sefaria/model/text.py @@ -25,7 +25,7 @@ import sefaria.system.cache as scache from sefaria.system.cache import in_memory_cache from sefaria.system.exceptions import InputError, BookNameError, PartialRefInputError, IndexSchemaError, \ - NoVersionFoundError, DictionaryEntryNotFoundError, MissingKeyError + NoVersionFoundError, DictionaryEntryNotFoundError, MissingKeyError, ComplexBookLevelRefError from sefaria.utils.hebrew import has_hebrew, is_all_hebrew, hebrew_term from sefaria.utils.util import list_depth, truncate_string from sefaria.datatype.jagged_array import JaggedTextArray, JaggedArray @@ -1693,7 +1693,7 @@ def __init__(self, oref, lang, vtitle, merge_versions=False, versions=None): elif oref.has_default_child(): #use default child: self.oref = oref.default_child_ref() else: - raise InputError("Can not get TextRange at this level, please provide a more precise reference") + raise ComplexBookLevelRefError(book_ref=oref.normal()) self.lang = lang self.vtitle = vtitle self.merge_versions = merge_versions @@ -2424,7 +2424,7 @@ def __init__(self, oref, context=1, commentary=True, version=None, lang=None, self._alts = [] if not isinstance(oref.index_node, JaggedArrayNode) and not oref.index_node.is_virtual: - raise InputError("Can not get TextFamily at this level, please provide a more precise reference") + raise InputError("Unable to find text for that ref") for i in range(0, context): oref = oref.context_ref() @@ -3991,7 +3991,7 @@ def padded_ref(self): if self.has_default_child(): return self.default_child_ref().padded_ref() elif self.is_book_level(): - raise InputError("Can not get TextRange at this level, please provide a more precise reference") + raise ComplexBookLevelRefError(book_ref=self.normal()) else: raise InputError("Cannot pad a schema node ref.") diff --git a/sefaria/system/exceptions.py b/sefaria/system/exceptions.py index d0029d2105..8ef21b8f59 100644 --- a/sefaria/system/exceptions.py +++ b/sefaria/system/exceptions.py @@ -100,8 +100,12 @@ def __init__(self, method): self.message = f"'{method}' is not a valid HTTP API method." super().__init__(self.message) -class ComplexBookLevelRefError(Exception): +class ComplexBookLevelRefError(InputError): def __init__(self, book_ref): self.book_ref = book_ref - self.message = f"Please pass a more specific ref for this book, and try again. You passed in a \'complex\' book-level ref. We only support book-level refs in cases of texts with a 'simple' structure. To learn more about the structure of a text on Sefaria, see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text" + self.message = (f"You passed '{book_ref}', please pass a more specific ref for this book, and try again. " + f"'{book_ref}' is a \'complex\' book-level ref. We only support book-level " + f"refs in cases of texts with a 'simple' structure. To learn more about the " + f"structure of a text on Sefaria, " + f"see: https://developers.sefaria.org/docs/the-schema-of-a-simple-text") super().__init__(self.message) From 1827d9db338d221cfe56ee073452ddd06ed43190 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Wed, 13 Nov 2024 14:04:15 +0200 Subject: [PATCH 020/151] refactor(topics): Refactor to use two models, Topic and TopicPool to represent many to many relationship --- admin_tools/migrations/0001_initial.py | 24 ----------- .../migrations/0002_delete_topicpoollink.py | 18 --------- admin_tools/migrations/0003_topicpoollink.py | 25 ------------ .../migrations/0004_auto_20241111_2328.py | 19 --------- admin_tools/models/__init__.py | 1 - admin_tools/models/topic_pool_link.py | 40 ------------------- sefaria/settings.py | 2 +- {admin_tools => topics}/__init__.py | 0 .../migrations/__init__.py | 0 topics/models/__init__.py | 2 + topics/models/pool.py | 15 +++++++ topics/models/topic.py | 12 ++++++ 12 files changed, 30 insertions(+), 128 deletions(-) delete mode 100644 admin_tools/migrations/0001_initial.py delete mode 100644 admin_tools/migrations/0002_delete_topicpoollink.py delete mode 100644 admin_tools/migrations/0003_topicpoollink.py delete mode 100644 admin_tools/migrations/0004_auto_20241111_2328.py delete mode 100644 admin_tools/models/__init__.py delete mode 100644 admin_tools/models/topic_pool_link.py rename {admin_tools => topics}/__init__.py (100%) rename {admin_tools => topics}/migrations/__init__.py (100%) create mode 100644 topics/models/__init__.py create mode 100644 topics/models/pool.py create mode 100644 topics/models/topic.py diff --git a/admin_tools/migrations/0001_initial.py b/admin_tools/migrations/0001_initial.py deleted file mode 100644 index ec43fcb95a..0000000000 --- a/admin_tools/migrations/0001_initial.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-11 17:45 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='TopicPoolLink', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('pool', models.CharField(max_length=255)), - ('topic_slug', models.CharField(max_length=255)), - ], - ), - ] diff --git a/admin_tools/migrations/0002_delete_topicpoollink.py b/admin_tools/migrations/0002_delete_topicpoollink.py deleted file mode 100644 index 98e95d6eef..0000000000 --- a/admin_tools/migrations/0002_delete_topicpoollink.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-11 18:42 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('admin_tools', '0001_initial'), - ] - - operations = [ - migrations.DeleteModel( - name='TopicPoolLink', - ), - ] diff --git a/admin_tools/migrations/0003_topicpoollink.py b/admin_tools/migrations/0003_topicpoollink.py deleted file mode 100644 index 95558d20a4..0000000000 --- a/admin_tools/migrations/0003_topicpoollink.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-11 18:43 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('admin_tools', '0002_delete_topicpoollink'), - ] - - operations = [ - migrations.CreateModel( - name='TopicPoolLink', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('pool', models.CharField(max_length=255)), - ('topic_slug', models.CharField(max_length=255)), - ], - ), - ] diff --git a/admin_tools/migrations/0004_auto_20241111_2328.py b/admin_tools/migrations/0004_auto_20241111_2328.py deleted file mode 100644 index 866e648b61..0000000000 --- a/admin_tools/migrations/0004_auto_20241111_2328.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2024-11-12 03:28 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('admin_tools', '0003_topicpoollink'), - ] - - operations = [ - migrations.AlterUniqueTogether( - name='topicpoollink', - unique_together=set([('pool', 'topic_slug')]), - ), - ] diff --git a/admin_tools/models/__init__.py b/admin_tools/models/__init__.py deleted file mode 100644 index 6eaf38f2f7..0000000000 --- a/admin_tools/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .topic_pool_link import TopicPoolLink diff --git a/admin_tools/models/topic_pool_link.py b/admin_tools/models/topic_pool_link.py deleted file mode 100644 index f7426cf872..0000000000 --- a/admin_tools/models/topic_pool_link.py +++ /dev/null @@ -1,40 +0,0 @@ -from django.db import models -from enum import Enum - - -class PoolType(Enum): - TEXTUAL = "textual" - SHEETS = "sheets" - PROMOTED = "promoted" - - -class TopicPoolLinkManager(models.Manager): - def get_random_topic_slugs(self, pool=None, limit=10) -> list[str]: - query_set = self.get_queryset() - if pool: - query_set = query_set.filter(pool=pool) - query_set = query_set.values('topic_slug').distinct().order_by('?')[:limit] - return [x['topic_slug'] for x in query_set] - - @staticmethod - def get_pools_by_topic_slug(topic_slug) -> list[str]: - query_set = TopicPoolLink.objects.filter(topic_slug=topic_slug).values('pool').distinct() - return [x['pool'] for x in query_set] - - -class TopicPoolLink(models.Model): - pool = models.CharField(max_length=255) - topic_slug = models.CharField(max_length=255) - objects = TopicPoolLinkManager() - - class Meta: - unique_together = ('pool', 'topic_slug') - - def __str__(self): - return f"{self.pool} <> {self.topic_slug}" - - - - - - diff --git a/sefaria/settings.py b/sefaria/settings.py index 1a425e5392..bdb6dd7460 100644 --- a/sefaria/settings.py +++ b/sefaria/settings.py @@ -144,7 +144,7 @@ 'reader', 'sourcesheets', 'sefaria.gauth', - 'admin_tools', + 'topics', 'captcha', 'django.contrib.admin', 'anymail', diff --git a/admin_tools/__init__.py b/topics/__init__.py similarity index 100% rename from admin_tools/__init__.py rename to topics/__init__.py diff --git a/admin_tools/migrations/__init__.py b/topics/migrations/__init__.py similarity index 100% rename from admin_tools/migrations/__init__.py rename to topics/migrations/__init__.py diff --git a/topics/models/__init__.py b/topics/models/__init__.py new file mode 100644 index 0000000000..3c756d8991 --- /dev/null +++ b/topics/models/__init__.py @@ -0,0 +1,2 @@ +from .topic import Topic +from .pool import TopicPool diff --git a/topics/models/pool.py b/topics/models/pool.py new file mode 100644 index 0000000000..3facbb6b90 --- /dev/null +++ b/topics/models/pool.py @@ -0,0 +1,15 @@ +from django.db import models +from enum import Enum + + +class PoolType(Enum): + TEXTUAL = "textual" + SHEETS = "sheets" + PROMOTED = "promoted" + + +class TopicPool(models.Model): + name = models.CharField(max_length=255, unique=True) + + def __str__(self): + return f"TopicPool('{self.name}')" diff --git a/topics/models/topic.py b/topics/models/topic.py new file mode 100644 index 0000000000..6bba6523d4 --- /dev/null +++ b/topics/models/topic.py @@ -0,0 +1,12 @@ +from django.db import models +from topics.models.pool import TopicPool + + +class Topic(models.Model): + slug = models.CharField(max_length=255, unique=True) + en_title = models.CharField(max_length=255, blank=True, default="") + he_title = models.CharField(max_length=255, blank=True, default="") + pools = models.ManyToManyField(TopicPool, related_name="topics") + + def __str__(self): + return f"Topic('{self.slug}')" From ad18ba188bb3d8a90e425f5ad19cecfdc0e93e32 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 12:12:42 +0200 Subject: [PATCH 021/151] feat(topics): admin interface for topics and topic pools --- topics/admin.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 topics/admin.py diff --git a/topics/admin.py b/topics/admin.py new file mode 100644 index 0000000000..189c98cfe7 --- /dev/null +++ b/topics/admin.py @@ -0,0 +1,72 @@ +from django.contrib import admin, messages +from django.db.models import BooleanField, Case, When +from topics.models import Topic, TopicPool +from topics.models.pool import PoolType + + +def create_add_to_specific_pool_action(pool_name): + def add_to_specific_pool(modeladmin, request, queryset): + try: + pool = TopicPool.objects.get(name=pool_name) + for topic in queryset: + topic.pools.add(pool) + modeladmin.message_user(request, f"Added {queryset.count()} topics to {pool.name}", messages.SUCCESS) + + except TopicPool.DoesNotExist: + modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR) + + add_to_specific_pool.short_description = f"Add selected topics to '{pool_name}' pool" + return add_to_specific_pool + + +class TopicAdmin(admin.ModelAdmin): + list_display = ('slug', 'en_title', 'he_title', 'is_in_pool_general', 'is_in_pool_torah_tab') + filter_horizontal = ('pools',) + readonly_fields = ('slug', 'en_title', 'he_title') + actions = [create_add_to_specific_pool_action(pool_name) for pool_name in (PoolType.GENERAL.value, PoolType.TORAH_TAB.value)] + + def get_queryset(self, request): + queryset = super().get_queryset(request) + return queryset.annotate( + in_pool_general=Case( + When(pools__name=PoolType.GENERAL.value, then=True), + default=False, + output_field=BooleanField() + ), + in_pool_torah_tab=Case( + When(pools__name=PoolType.TORAH_TAB.value, then=True), + default=False, + output_field=BooleanField() + ) + ) + + def is_in_pool_general(self, obj): + return obj.in_pool_general + is_in_pool_general.boolean = True + is_in_pool_general.short_description = "General?" + is_in_pool_general.admin_order_field = 'in_pool_general' + + def is_in_pool_torah_tab(self, obj): + return obj.in_pool_torah_tab + is_in_pool_torah_tab.boolean = True + is_in_pool_torah_tab.short_description = "TorahTab?" + is_in_pool_torah_tab.admin_order_field = 'in_pool_torah_tab' + + +class TopicPoolAdmin(admin.ModelAdmin): + list_display = ('name', 'topic_names') + filter_horizontal = ('topics',) + readonly_fields = ('name',) + + def topic_names(self, obj): + topic_slugs = obj.topics.all().values_list('slug', flat=True) + str_rep = ', '.join(topic_slugs[:30]) + if len(topic_slugs) > 30: + str_rep = str_rep + '...' + return str_rep + + +admin.site.register(Topic, TopicAdmin) +admin.site.register(TopicPool, TopicPoolAdmin) + + From 9725b6162e3defb2ced57a9f6cd417992a733ed1 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 12:15:47 +0200 Subject: [PATCH 022/151] feat(topics): only show library topics in topic admin view --- topics/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topics/admin.py b/topics/admin.py index 189c98cfe7..8b8e2ce685 100644 --- a/topics/admin.py +++ b/topics/admin.py @@ -38,7 +38,7 @@ def get_queryset(self, request): default=False, output_field=BooleanField() ) - ) + ).filter(pools__name=PoolType.LIBRARY.value) def is_in_pool_general(self, obj): return obj.in_pool_general From 544df751865d4d38271f16698985ceb7055f6583 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:42:43 +0200 Subject: [PATCH 023/151] chore(topics): update pools migration to fully migrate --- .../migrate_good_to_promote_to_topic_pools.py | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/scripts/migrations/migrate_good_to_promote_to_topic_pools.py b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py index 1259dfa4df..c74e2b9ec3 100644 --- a/scripts/migrations/migrate_good_to_promote_to_topic_pools.py +++ b/scripts/migrations/migrate_good_to_promote_to_topic_pools.py @@ -1,14 +1,80 @@ import django +from django.db import IntegrityError + django.setup() -from sefaria.model import * -from admin_tools.models.topic_pool_link import PoolType, TopicPoolLink +from sefaria.model import TopicSet, RefTopicLinkSet +from topics.models.topic import Topic +from topics.models.pool import TopicPool, PoolType -def run(): +def add_to_torah_tab_pool(): + print('Adding topics to torah tab pool') + pool = TopicPool.objects.get(name=PoolType.TORAH_TAB.value) ts = TopicSet({'good_to_promote': True}) for topic in ts: - link = TopicPoolLink(topic_slug=topic.slug, pool=PoolType.PROMOTED.value) - link.save() + t = Topic.objects.get(slug=topic.slug) + t.pools.add(pool) + + +def add_to_library_pool(): + print('Adding topics to library pool') + pool = TopicPool.objects.get(name=PoolType.LIBRARY.value) + ts = TopicSet({'subclass': 'author'}) + for topic in ts: + t = Topic.objects.get(slug=topic.slug) + t.pools.add(pool) + links = RefTopicLinkSet({'is_sheet': False, 'linkType': 'about'}) + topic_slugs = {link.toTopic for link in links} + for slug in topic_slugs: + try: + t = Topic.objects.get(slug=slug) + t.pools.add(pool) + except Topic.DoesNotExist: + print('Could not find topic with slug {}'.format(slug)) + + +def add_to_sheets_pool(): + print('Adding topics to sheets pool') + pool = TopicPool.objects.get(name=PoolType.SHEETS.value) + links = RefTopicLinkSet({'is_sheet': True, 'linkType': 'about'}) + topic_slugs = {link.toTopic for link in links} + for slug in topic_slugs: + try: + t = Topic.objects.get(slug=slug) + t.pools.add(pool) + except Topic.DoesNotExist: + print('Could not find topic with slug {}'.format(slug)) + + +def delete_all_data(): + print("Delete data") + Topic.pools.through.objects.all().delete() + Topic.objects.all().delete() + TopicPool.objects.all().delete() + + +def add_topics(): + print('Adding topics') + for topic in TopicSet({}): + try: + Topic.objects.create(slug=topic.slug, en_title=topic.get_primary_title('en'), he_title=topic.get_primary_title('he')) + except IntegrityError: + print('Duplicate topic', topic.slug) + + +def add_pools(): + print('Adding pools') + for pool_name in [PoolType.LIBRARY.value, PoolType.SHEETS.value, PoolType.GENERAL.value, PoolType.TORAH_TAB.value]: + TopicPool.objects.create(name=pool_name) + + +def run(): + delete_all_data() + add_topics() + add_pools() + add_to_torah_tab_pool() + add_to_library_pool() + add_to_sheets_pool() if __name__ == "__main__": From eed87c609768125f43e34c1fedf38080e4dd664b Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:43:02 +0200 Subject: [PATCH 024/151] feat(topics): add filters and boolean columns --- topics/admin.py | 71 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/topics/admin.py b/topics/admin.py index 8b8e2ce685..a0e765ecba 100644 --- a/topics/admin.py +++ b/topics/admin.py @@ -1,11 +1,10 @@ from django.contrib import admin, messages -from django.db.models import BooleanField, Case, When from topics.models import Topic, TopicPool from topics.models.pool import PoolType -def create_add_to_specific_pool_action(pool_name): - def add_to_specific_pool(modeladmin, request, queryset): +def create_add_to_pool_action(pool_name): + def add_to_pool(modeladmin, request, queryset): try: pool = TopicPool.objects.get(name=pool_name) for topic in queryset: @@ -15,42 +14,70 @@ def add_to_specific_pool(modeladmin, request, queryset): except TopicPool.DoesNotExist: modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR) - add_to_specific_pool.short_description = f"Add selected topics to '{pool_name}' pool" - return add_to_specific_pool + add_to_pool.short_description = f"Add selected topics to '{pool_name}' pool" + add_to_pool.__name__ = f"add_to_specific_pool_{pool_name}" + return add_to_pool + + +def create_remove_from_pool_action(pool_name): + def remove_from_pool(modeladmin, request, queryset): + try: + pool = TopicPool.objects.get(name=pool_name) + for topic in queryset: + topic.pools.remove(pool) + modeladmin.message_user(request, f"Removed {queryset.count()} topics from {pool.name}", messages.SUCCESS) + + except TopicPool.DoesNotExist: + modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR) + + remove_from_pool.short_description = f"Remove selected topics from '{pool_name}' pool" + remove_from_pool.__name__ = f"remove_from_pool_{pool_name}" + return remove_from_pool + + +class PoolFilter(admin.SimpleListFilter): + title = 'Pool Filter' + parameter_name = 'pool' + + def lookups(self, request, model_admin): + return [ + (PoolType.GENERAL.value, 'General Pool'), + (PoolType.TORAH_TAB.value, 'TorahTab Pool'), + ] + + def queryset(self, request, queryset): + pool_name = self.value() + if pool_name: + pool = TopicPool.objects.get(name=pool_name) + return queryset.filter(pools=pool) + return queryset class TopicAdmin(admin.ModelAdmin): list_display = ('slug', 'en_title', 'he_title', 'is_in_pool_general', 'is_in_pool_torah_tab') + list_filter = (PoolFilter,) filter_horizontal = ('pools',) readonly_fields = ('slug', 'en_title', 'he_title') - actions = [create_add_to_specific_pool_action(pool_name) for pool_name in (PoolType.GENERAL.value, PoolType.TORAH_TAB.value)] + actions = [ + create_add_to_pool_action(PoolType.GENERAL.value), + create_add_to_pool_action(PoolType.TORAH_TAB.value), + create_remove_from_pool_action(PoolType.GENERAL.value), + create_remove_from_pool_action(PoolType.TORAH_TAB.value), + ] def get_queryset(self, request): queryset = super().get_queryset(request) - return queryset.annotate( - in_pool_general=Case( - When(pools__name=PoolType.GENERAL.value, then=True), - default=False, - output_field=BooleanField() - ), - in_pool_torah_tab=Case( - When(pools__name=PoolType.TORAH_TAB.value, then=True), - default=False, - output_field=BooleanField() - ) - ).filter(pools__name=PoolType.LIBRARY.value) + return queryset.filter(pools__name=PoolType.LIBRARY.value) def is_in_pool_general(self, obj): - return obj.in_pool_general + return obj.pools.filter(name=PoolType.GENERAL.value).exists() is_in_pool_general.boolean = True is_in_pool_general.short_description = "General?" - is_in_pool_general.admin_order_field = 'in_pool_general' def is_in_pool_torah_tab(self, obj): - return obj.in_pool_torah_tab + return obj.pools.filter(name=PoolType.TORAH_TAB.value).exists() is_in_pool_torah_tab.boolean = True is_in_pool_torah_tab.short_description = "TorahTab?" - is_in_pool_torah_tab.admin_order_field = 'in_pool_torah_tab' class TopicPoolAdmin(admin.ModelAdmin): From e85ecf1f117a4b30e277cb6f11998d58a94e5310 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:51:59 +0200 Subject: [PATCH 025/151] refactor(topics): refactor sefaria functions to use new django models --- reader/views.py | 2 +- sefaria/helper/topic.py | 4 ++-- sefaria/model/topic.py | 21 ++++++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/reader/views.py b/reader/views.py index 1395cf1ac2..dff1551aa5 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4229,7 +4229,7 @@ def random_by_topic_api(request): """ Returns Texts API data for a random text taken from popular topic tags """ - from admin_tools.models.topic_pool_link import PoolType + from topics.models.topic_pool_link import PoolType cb = request.GET.get("callback", None) random_topic = get_random_topic(PoolType.PROMOTED.value) if random_topic is None: diff --git a/sefaria/helper/topic.py b/sefaria/helper/topic.py index d69e3aa0cd..e7af0f6836 100644 --- a/sefaria/helper/topic.py +++ b/sefaria/helper/topic.py @@ -285,8 +285,8 @@ def get_random_topic(pool=None) -> Optional[Topic]: :param pool: name of the pool from which to select the topic. If `None`, all topics are considered. :return: Returns a random topic from the database. If you provide `pool`, then the selection is limited to topics in that pool. """ - from admin_tools.models import TopicPoolLink - random_topic_slugs = TopicPoolLink.objects.get_random_topic_slugs(pool=pool, limit=1) + from topics.models import Topic as DjangoTopic + random_topic_slugs = DjangoTopic.objects.sample_topic_slugs('random', pool, limit=1) if len(random_topic_slugs) == 0: return None diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index c126c8b2d0..d28ca42e45 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -5,7 +5,8 @@ from .schema import AbstractTitledObject, TitleGroup from .text import Ref, IndexSet, AbstractTextRecord, Index, Term from .category import Category -from admin_tools.models.topic_pool_link import TopicPoolLink, PoolType +from topics.models import Topic as DjangoTopic +from topics.models import TopicPool, PoolType from sefaria.system.exceptions import InputError, DuplicateRecordError from sefaria.model.timeperiod import TimePeriod, LifePeriod from sefaria.system.validators import validate_url @@ -188,7 +189,7 @@ def load(self, query, proj=None): def _set_derived_attributes(self): self.set_titles(getattr(self, "titles", None)) - self.pools = TopicPoolLink.objects.get_pools_by_topic_slug(getattr(self, "slug", None)) + self.pools = DjangoTopic.objects.get_pools_by_topic_slug(getattr(self, "slug", None)) if self.__class__ != Topic and not getattr(self, "subclass", False): # in a subclass. set appropriate "subclass" attribute setattr(self, "subclass", self.reverse_subclass_map[self.__class__.__name__]) @@ -228,17 +229,15 @@ def get_pools(self) -> list[str]: def has_pool(self, pool: str) -> bool: return pool in self.get_pools() - def add_pool(self, pool: str) -> None: - try: - link = TopicPoolLink(pool=pool, topic_slug=self.slug) - link.save() - except IntegrityError: - raise DuplicateRecordError(f"'{pool}'<>'{self.slug}' link already exists in TopicPoolLink table.") + def add_pool(self, pool_name: str) -> None: + pool = TopicPool.objects.get(name=pool_name) + DjangoTopic.objects.get(slug=self.slug).pools.add(pool) self.pools = self.get_pools() - self.pools.append(pool) + self.pools.append(pool_name) def remove_pool(self, pool) -> None: - TopicPoolLink.objects.filter(pool=pool, topic_slug=self.slug).delete() + pool = TopicPool.objects.get(name=pool) + DjangoTopic.objects.get(slug=self.slug).pools.remove(pool) pools = self.get_pools() pools.remove(pool) @@ -959,7 +958,7 @@ def set_description(self, lang, title, prompt): return self def get_related_pool(self): - return PoolType.SHEETS.value if self.is_sheet else PoolType.TEXTUAL.value + return PoolType.SHEETS.value if self.is_sheet else PoolType.LIBRARY.value def get_topic(self): return Topic().load({'slug': self.toTopic}) From b4837142dcda2963d4f8ac6dfa9d781027879848 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:54:23 +0200 Subject: [PATCH 026/151] chore(topics): add topic migrations --- topics/migrations/0001_initial.py | 37 ++++++++++++++++++++ topics/migrations/0002_auto_20241113_0809.py | 20 +++++++++++ 2 files changed, 57 insertions(+) create mode 100644 topics/migrations/0001_initial.py create mode 100644 topics/migrations/0002_auto_20241113_0809.py diff --git a/topics/migrations/0001_initial.py b/topics/migrations/0001_initial.py new file mode 100644 index 0000000000..86d8cb24f2 --- /dev/null +++ b/topics/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-13 12:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Topic', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('slug', models.CharField(max_length=255, unique=True)), + ('en_title', models.CharField(blank=True, default='', max_length=255)), + ('he_title', models.CharField(blank=True, default='', max_length=255)), + ], + ), + migrations.CreateModel( + name='TopicPool', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ], + ), + migrations.AddField( + model_name='topic', + name='pools', + field=models.ManyToManyField(related_name='topics', to='topics.TopicPool'), + ), + ] diff --git a/topics/migrations/0002_auto_20241113_0809.py b/topics/migrations/0002_auto_20241113_0809.py new file mode 100644 index 0000000000..4fff2f2c79 --- /dev/null +++ b/topics/migrations/0002_auto_20241113_0809.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-11-13 12:09 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('topics', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='topic', + name='pools', + field=models.ManyToManyField(blank=True, related_name='topics', to='topics.TopicPool'), + ), + ] From fb18fcd1e5570a63584799c5c5e0b57ed7ab3ddc Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:54:35 +0200 Subject: [PATCH 027/151] chore(topics): add PoolType to model export --- topics/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topics/models/__init__.py b/topics/models/__init__.py index 3c756d8991..4c01d93533 100644 --- a/topics/models/__init__.py +++ b/topics/models/__init__.py @@ -1,2 +1,2 @@ from .topic import Topic -from .pool import TopicPool +from .pool import TopicPool, PoolType From f67db0858bfaae82afc0f325a22882ed2bfa9e65 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:54:54 +0200 Subject: [PATCH 028/151] refactor(topics): rename pools --- topics/models/pool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/topics/models/pool.py b/topics/models/pool.py index 3facbb6b90..b84df46fec 100644 --- a/topics/models/pool.py +++ b/topics/models/pool.py @@ -3,9 +3,10 @@ class PoolType(Enum): - TEXTUAL = "textual" + LIBRARY = "library" SHEETS = "sheets" - PROMOTED = "promoted" + TORAH_TAB = "torah_tab" + GENERAL = "general" class TopicPool(models.Model): From 67dec73208b81ce51ca30c7bd7f192b59dbc8c03 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 14:55:21 +0200 Subject: [PATCH 029/151] feat(topics): add utility funcs to topic model --- topics/models/topic.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/topics/models/topic.py b/topics/models/topic.py index 6bba6523d4..9613518ace 100644 --- a/topics/models/topic.py +++ b/topics/models/topic.py @@ -1,12 +1,32 @@ from django.db import models +import random from topics.models.pool import TopicPool +class TopicManager(models.Manager): + def sample_topic_slugs(self, order, pool: str = None, limit=10) -> list[str]: + if pool: + topics = self.get_topic_slugs_by_pool(pool) + else: + topics = self.all().values_list('slug', flat=True) + if order == 'random': + return random.sample(list(topics), min(limit, len(topics))) + else: + raise Exception("Invalid order: '{}'".format(order)) + + def get_pools_by_topic_slug(self, topic_slug: str) -> list[str]: + return self.filter(topic_slug=topic_slug).values_list("pools__name", flat=True) + + def get_topic_slugs_by_pool(self, pool: str) -> list[str]: + return self.filter(pools__name=pool).values_list("slug", flat=True) + + class Topic(models.Model): slug = models.CharField(max_length=255, unique=True) en_title = models.CharField(max_length=255, blank=True, default="") he_title = models.CharField(max_length=255, blank=True, default="") - pools = models.ManyToManyField(TopicPool, related_name="topics") + pools = models.ManyToManyField(TopicPool, related_name="topics", blank=True) + objects = TopicManager() def __str__(self): return f"Topic('{self.slug}')" From 86804eb2ca6130fbf8b268ec1020b490a98a33bc Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 15:51:48 +0200 Subject: [PATCH 030/151] fix(topics): remove pools from mongo topics model --- sefaria/model/topic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index d28ca42e45..c729d83a02 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -158,7 +158,6 @@ class Topic(abst.SluggedAbstractMongoRecord, AbstractTitledObject): "data_source", #any topic edited manually should display automatically in the TOC and this flag ensures this 'image', "portal_slug", # slug to relevant Portal object - 'pools', # list of strings, any of them represents a pool that this topic is member of ] attr_schemas = { From 17c6a31fc50417f14728d7f370df4eb9ab30bcd9 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 15:52:11 +0200 Subject: [PATCH 031/151] fix(topics): fix query --- topics/models/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/topics/models/topic.py b/topics/models/topic.py index 9613518ace..b0211a5def 100644 --- a/topics/models/topic.py +++ b/topics/models/topic.py @@ -15,7 +15,7 @@ def sample_topic_slugs(self, order, pool: str = None, limit=10) -> list[str]: raise Exception("Invalid order: '{}'".format(order)) def get_pools_by_topic_slug(self, topic_slug: str) -> list[str]: - return self.filter(topic_slug=topic_slug).values_list("pools__name", flat=True) + return self.filter(slug=topic_slug).values_list("pools__name", flat=True) def get_topic_slugs_by_pool(self, pool: str) -> list[str]: return self.filter(pools__name=pool).values_list("slug", flat=True) From b2682468cbfb02e49e2c6697a8122dcd9b1d8767 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 15:53:06 +0200 Subject: [PATCH 032/151] refactor(topics): import and pool name --- reader/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reader/views.py b/reader/views.py index dff1551aa5..e4ca937670 100644 --- a/reader/views.py +++ b/reader/views.py @@ -4229,9 +4229,9 @@ def random_by_topic_api(request): """ Returns Texts API data for a random text taken from popular topic tags """ - from topics.models.topic_pool_link import PoolType + from topics.models import PoolType cb = request.GET.get("callback", None) - random_topic = get_random_topic(PoolType.PROMOTED.value) + random_topic = get_random_topic(PoolType.TORAH_TAB.value) if random_topic is None: return random_by_topic_api(request) random_source = get_random_topic_source(random_topic) From 30736ee4ee1e426920594de1df73427fdb421aea Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:05:02 +0200 Subject: [PATCH 033/151] chore(topics): update django topic model on mongo topic save --- sefaria/model/topic.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index c729d83a02..1faaea7a06 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -193,6 +193,13 @@ def _set_derived_attributes(self): # in a subclass. set appropriate "subclass" attribute setattr(self, "subclass", self.reverse_subclass_map[self.__class__.__name__]) + def _pre_save(self): + super()._pre_save() + django_topic, created = DjangoTopic.objects.get_or_create(slug=self.slug) + django_topic.en_title = self.get_primary_title('en') + django_topic.he_title = self.get_primary_title('he') + django_topic.save() + def _validate(self): super(Topic, self)._validate() if getattr(self, 'subclass', False): From 53affe9e19e055853014f344b4d4f8dd510bff9c Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:16:56 +0200 Subject: [PATCH 034/151] chore(topics): update django topic when mongo topic slug changes --- sefaria/model/topic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index 1faaea7a06..edf4c8411c 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -389,6 +389,7 @@ def set_slug(self, new_slug) -> None: old_slug = getattr(self, slug_field) setattr(self, slug_field, new_slug) setattr(self, slug_field, self.normalize_slug_field(slug_field)) + DjangoTopic.objects.filter(slug=old_slug).update(slug=new_slug) self.save() # so that topic with this slug exists when saving links to it self.merge(old_slug) @@ -464,6 +465,7 @@ def merge(self, other: Union['Topic', str]) -> None: setattr(self, attr, getattr(other, attr)) self.save() other.delete() + DjangoTopic.objects.get(slug=other_slug).delete() def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): """ From f754481f342a528701df2347f4243966ab0fcf84 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:17:27 +0200 Subject: [PATCH 035/151] chore(topics): remove extra newline --- sefaria/model/topic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index edf4c8411c..e211379e37 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -393,7 +393,6 @@ def set_slug(self, new_slug) -> None: self.save() # so that topic with this slug exists when saving links to it self.merge(old_slug) - def merge(self, other: Union['Topic', str]) -> None: """ Merge `other` into `self`. This means that all data from `other` will be merged into self. From d787bf66b1037b54756a528eeae3fa3b772f6ce0 Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:22:00 +0200 Subject: [PATCH 036/151] refactor(topics): move delete to Topic delete dependency --- sefaria/model/topic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sefaria/model/topic.py b/sefaria/model/topic.py index e211379e37..5d23144241 100644 --- a/sefaria/model/topic.py +++ b/sefaria/model/topic.py @@ -464,7 +464,6 @@ def merge(self, other: Union['Topic', str]) -> None: setattr(self, attr, getattr(other, attr)) self.save() other.delete() - DjangoTopic.objects.get(slug=other_slug).delete() def link_set(self, _class='intraTopic', query_kwargs: dict = None, **kwargs): """ @@ -1170,6 +1169,7 @@ def process_topic_delete(topic): for sheet in db.sheets.find({"topics.slug": topic.slug}): sheet["topics"] = [t for t in sheet["topics"] if t["slug"] != topic.slug] db.sheets.save(sheet) + DjangoTopic.objects.get(slug=topic.slug).delete() def process_topic_description_change(topic, **kwargs): """ From af9f31d0b831676803d0712605526539fd8f396c Mon Sep 17 00:00:00 2001 From: nsantacruz Date: Thu, 14 Nov 2024 16:44:58 +0200 Subject: [PATCH 037/151] test(topics): add tests to make sure django topic remains in sync with mongo topic --- sefaria/model/tests/topic_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sefaria/model/tests/topic_test.py b/sefaria/model/tests/topic_test.py index 56345624f1..4d61ccab5c 100644 --- a/sefaria/model/tests/topic_test.py +++ b/sefaria/model/tests/topic_test.py @@ -3,6 +3,7 @@ from sefaria.model.text import Ref from sefaria.system.database import db from sefaria.system.exceptions import SluggedMongoRecordMissingError +from topics.models import Topic as DjangoTopic from sefaria.helper.topic import update_topic @@ -155,6 +156,22 @@ def test_merge(self, topic_graph_to_merge): {"slug": '30', 'asTyped': 'thirty'} ] + t40 = Topic.init('40') + assert t40 is None + DjangoTopic.objects.get(slug='20') + with pytest.raises(DjangoTopic.DoesNotExist): + DjangoTopic.objects.get(slug='40') + + def test_change_title(self, topic_graph): + ts = topic_graph['topics'] + dt1 = DjangoTopic.objects.get(slug=ts['1'].slug) + assert dt1.en_title == ts['1'].get_primary_title('en') + ts['1'].title_group.add_title('new title', 'en', True, True) + ts['1'].save() + dt1 = DjangoTopic.objects.get(slug=ts['1'].slug) + assert dt1.en_title == ts['1'].get_primary_title('en') + + def test_sanitize(self): t = Topic() t.slug = "sdfsdg