Modularization main2

.github/workflows/weekly.yaml

@@ -3,6 +3,7 @@ name: Weekly Tasks
 on:
   schedule:
     - cron: '0 0 * 6 *'
+  workflow_dispatch:
 
 jobs:
   build-mongo:
@@ -11,11 +12,21 @@ jobs:
       contents: 'read'
       id-token: 'write'
     steps:
+      - name: Maximize build space
+        uses: AdityaGarg8/[email protected]
+        with:
+          remove-android: 'true'
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - uses: actions/checkout@v2
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config: ./build/standalone-db/buildkit.toml
       - id: auth
         name: Authenticate to Google Cloud
         uses: google-github-actions/auth@v1
@@ -37,27 +48,26 @@ jobs:
           password: '${{ steps.auth.outputs.access_token }}'
       - name: Get current date
         id: date
-        run: echo "date$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
+        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
       - name: Generate image metadata
         id: meta
         uses: docker/metadata-action@v3
         with:
           images: |
             gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-mongo
-            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
+            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/sefaria-public/sefaria-mongo
           # generate Docker tags based on the following events/attributes
           tags: |
             type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
             type=sha
           flavor: |
             latest=true
       - name: build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
-          cache-from: type=registry, ref=sefaria-mongo/cache
-          cache-to: type=registry, ref=sefaria-mongo/cache, mode=max
           context: .
           push: true
+          platforms: linux/amd64,linux/arm64
           file: ./build/standalone-db/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

api/api_errors.py

@@ -0,0 +1,16 @@
+"""
+Classes for API errors
+"""
+from sefaria.client.util import jsonResponse
+
+
+class APIInvalidInputException(Exception):
+    """
+    When data in an invalid format is passed to an API
+    """
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = message
+
+    def to_json_response(self):
+        return jsonResponse({"invalid_input_error": self.message}, status=400)

api/views.py

@@ -1,6 +1,7 @@
 from sefaria.model import *
 from sefaria.model.text_reuqest_adapter import TextRequestAdapter
 from sefaria.client.util import jsonResponse
+from sefaria.system.exceptions import InputError, ComplexBookLevelRefError
 from django.views import View
 from .api_warnings import *
 
@@ -53,6 +54,12 @@ def get(self, request, *args, **kwargs):
         if return_format not in self.RETURN_FORMATS:
             return jsonResponse({'error': f'return_format should be one of those formats: {self.RETURN_FORMATS}.'}, status=400)
         text_manager = TextRequestAdapter(self.oref, versions_params, fill_in_missing_segments, return_format)
-        data = text_manager.get_versions_for_query()
-        data = self._handle_warnings(data)
+
+        try:
+            data = text_manager.get_versions_for_query()
+            data = self._handle_warnings(data)
+
+        except Exception as e:
+            return jsonResponse({'error': str(e)}, status=400)
+
         return jsonResponse(data)

build/standalone-db/buildkit.toml

@@ -0,0 +1,2 @@
+[worker.oci]
+  max-parallelism = 1

django_topics/README.md

@@ -0,0 +1,8 @@
+# Django Topics app
+
+Django app that defines models and admin interfaces for editing certain aspects of topics that are unique to Sefaria's product and not needed for general usage of Sefaria's data.
+
+Currently contains methods to:
+- Edit which topics are in which pools
+- Define topic of the day schedule
+- Define seasonal topic schedule

django_topics/admin.py

@@ -0,0 +1,229 @@
+from django.contrib import admin, messages
+from django.utils.html import format_html
+from django_topics.models import Topic, TopicPool, TopicOfTheDayEnglish, TopicOfTheDayHebrew, SeasonalTopicEnglish, SeasonalTopicHebrew
+from django_topics.models.pool import PoolType
+
+
+def create_add_to_pool_action(pool_name):
+    def add_to_pool(modeladmin, request, queryset):
+        try:
+            pool = TopicPool.objects.get(name=pool_name)
+            for topic in queryset:
+                topic.pools.add(pool)
+            modeladmin.message_user(request, f"Added {queryset.count()} topics to {pool.name}", messages.SUCCESS)
+
+        except TopicPool.DoesNotExist:
+            modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR)
+
+    add_to_pool.short_description = f"Add selected topics to '{pool_name}' pool"
+    add_to_pool.__name__ = f"add_to_specific_pool_{pool_name}"
+    return add_to_pool
+
+
+def create_remove_from_pool_action(pool_name):
+    def remove_from_pool(modeladmin, request, queryset):
+        try:
+            pool = TopicPool.objects.get(name=pool_name)
+            for topic in queryset:
+                topic.pools.remove(pool)
+            modeladmin.message_user(request, f"Removed {queryset.count()} topics from {pool.name}", messages.SUCCESS)
+
+        except TopicPool.DoesNotExist:
+            modeladmin.message_user(request, "The specified pool does not exist.", messages.ERROR)
+
+    remove_from_pool.short_description = f"Remove selected topics from '{pool_name}' pool"
+    remove_from_pool.__name__ = f"remove_from_pool_{pool_name}"
+    return remove_from_pool
+
+
+class PoolFilter(admin.SimpleListFilter):
+    title = 'Pool Filter'
+    parameter_name = 'pool'
+
+    def lookups(self, request, model_admin):
+        return [
+            ('general_en', 'General Pool EN'),
+            ('general_he', 'General Pool HE'),
+            (PoolType.TORAH_TAB.value, 'TorahTab Pool'),
+        ]
+
+    def queryset(self, request, queryset):
+        pool_name = self.value()
+        if pool_name:
+            pool = TopicPool.objects.get(name=pool_name)
+            return queryset.filter(pools=pool)
+        return queryset
+
+
+@admin.register(Topic)
+class TopicAdmin(admin.ModelAdmin):
+    list_display = ('slug', 'en_title', 'he_title', 'is_in_pool_general_en', 'is_in_pool_general_he', 'is_in_pool_torah_tab', 'sefaria_link')
+    list_filter = (PoolFilter,)
+    filter_horizontal = ('pools',)
+    search_fields = ('slug', 'en_title', 'he_title')
+    readonly_fields = ('slug', 'en_title', 'he_title')
+    actions = [
+        create_add_to_pool_action('general_en'),
+        create_add_to_pool_action('general_he'),
+        create_add_to_pool_action(PoolType.TORAH_TAB.value),
+        create_remove_from_pool_action('general_en'),
+        create_remove_from_pool_action('general_he'),
+        create_remove_from_pool_action(PoolType.TORAH_TAB.value),
+    ]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def get_queryset(self, request):
+        queryset = super().get_queryset(request)
+        return queryset.filter(pools__name=PoolType.LIBRARY.value)
+
+    def is_in_pool_general_en(self, obj):
+        return obj.pools.filter(name='general_en').exists()
+    is_in_pool_general_en.boolean = True
+    is_in_pool_general_en.short_description = "General Pool EN"
+
+    def is_in_pool_general_he(self, obj):
+        return obj.pools.filter(name='general_he').exists()
+    is_in_pool_general_he.boolean = True
+    is_in_pool_general_he.short_description = "General Pool HE"
+
+    def is_in_pool_torah_tab(self, obj):
+        return obj.pools.filter(name=PoolType.TORAH_TAB.value).exists()
+    is_in_pool_torah_tab.boolean = True
+    is_in_pool_torah_tab.short_description = "TorahTab Pool"
+
+    def sefaria_link(self, obj):
+        url = f"https://www.sefaria.org/topics/{obj.slug}"
+        return format_html('<a href="{}" target="_blank">{}</a>', url, obj.slug)
+    sefaria_link.short_description = "Sefaria Link"
+
+
+class TopicOfTheDayAdmin(admin.ModelAdmin):
+    exclude = ("lang",)  # not for manual editing
+    list_display = ('start_date', 'topic')
+    list_filter = ('start_date',)
+    raw_id_fields = ('topic',)
+    search_fields = ('topic__slug', 'topic__en_title', 'topic__he_title')
+    date_hierarchy = 'start_date'
+    ordering = ['-start_date']
+    fieldsets = (
+        (None, {
+            'fields': ('topic', 'start_date'),
+        }),
+    )
+
+    def formfield_for_foreignkey(self, db_field, request, **kwargs):
+        if db_field.name == "topic":
+            kwargs["label"] = "Topic slug"
+            kwargs["help_text"] = "Use the magnifying glass button to select a topic."
+        return super().formfield_for_foreignkey(db_field, request, **kwargs)
+
+
+@admin.register(TopicOfTheDayEnglish)
+class TopicOfTheDayAdminEnglish(TopicOfTheDayAdmin):
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.filter(lang="en")
+
+
+@admin.register(TopicOfTheDayHebrew)
+class TopicOfTheDayAdminHebrew(TopicOfTheDayAdmin):
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.filter(lang="he")
+
+
+class SeasonalTopicAdmin(admin.ModelAdmin):
+    exclude = ("lang",)  # not for manual editing
+    list_display = (
+        'start_date',
+        'topic',
+        'display_date_prefix',
+        'display_date_suffix',
+        'secondary_topic',
+        'display_start_date_israel',
+        'display_end_date_israel',
+        'display_start_date_diaspora',
+        'display_end_date_diaspora'
+    )
+    raw_id_fields = ('topic', 'secondary_topic')
+    list_filter = (
+        'start_date',
+        'display_start_date_israel',
+        'display_start_date_diaspora'
+    )
+    ordering = ['-start_date']
+    search_fields = ('topic__slug', 'topic__en_title', 'topic__he_title', 'secondary_topic__slug')
+    autocomplete_fields = ('topic', 'secondary_topic')
+    date_hierarchy = 'start_date'
+    fieldsets = (
+        (None, {
+            'fields': (
+                'topic',
+                'secondary_topic',
+                'start_date'
+            )
+        }),
+        ('Display Date Prefix/Suffix', {
+            'fields': (
+                'display_date_prefix',
+                'display_date_suffix',
+            ),
+            'description': 'Prefix/Suffix that will be displayed around the secondary topic.',
+        }),
+        ('Israel Display Dates', {
+            'fields': (
+                'display_start_date_israel',
+                'display_end_date_israel'
+            ),
+            'description': 'Dates to be displayed to the user of when this topic is "happening". '
+                           'E.g. for a holiday, when the holiday occurs. '
+                           'When the dates are the same for both Israel and Diaspora, only fill out Israeli dates. '
+                           'Similarly, when the start and end dates are the same, only fill out start date.'
+        }),
+        ('Diaspora Display Dates', {
+            'fields': (
+                'display_start_date_diaspora',
+                'display_end_date_diaspora'
+            ),
+            'description': 'When the dates are the same for both Israel and Diaspora, only fill out Israeli dates. '
+                           'Similarly, when the start and end dates are the same, only fill out start date.'
+
+        }),
+    )
+
+    def formfield_for_foreignkey(self, db_field, request, **kwargs):
+        if db_field.name == "topic":
+            kwargs["label"] = "Topic slug"
+            kwargs["help_text"] = "Use the magnifying glass button to select a topic."
+        if db_field.name == "secondary_topic":
+            kwargs["label"] = "Secondary topic slug"
+        return super().formfield_for_foreignkey(db_field, request, **kwargs)
+
+    def save_model(self, request, obj, form, change):
+        """
+        Overriding the save_model to ensure the model's clean method is executed.
+        """
+        obj.clean()
+        super().save_model(request, obj, form, change)
+
+
+@admin.register(SeasonalTopicEnglish)
+class SeasonalTopicAdminEnglish(SeasonalTopicAdmin):
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.filter(lang="en")
+
+
+@admin.register(SeasonalTopicHebrew)
+class SeasonalTopicAdminHebrew(SeasonalTopicAdmin):
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.filter(lang="he")

django_topics/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class DjangoTopicsAppConfig(AppConfig):
+    name = "django_topics"
+    verbose_name = "Topics Management"