Merge pull request #40 from chnm/gazetteer

feat: Implementation of the gazetteer index

config/settings.py

@@ -52,6 +52,8 @@
     "django.contrib.contenttypes",
     "django.contrib.sessions",
     "django.contrib.staticfiles",
+    "rest_framework",
+    "fontawesomefree",
     "tailwind",
     "theme",
     "django_browser_reload",

config/urls.py

@@ -1,3 +1,5 @@
+from django.conf import settings
+from django.conf.urls.static import static
 from django.contrib import admin
 from django.contrib.auth import views as auth_views
 from django.urls import include, path
@@ -6,8 +8,9 @@
     path("", include("manuscript.urls")),
     path("accounts/", include("allauth.urls")),
     path("admin/", admin.site.urls),
+    path("toponyms/", include(("manuscript.urls", "toponyms"), namespace="toponyms")),
     path("login/", auth_views.LoginView.as_view(), name="login"),
     path("logout/", auth_views.LogoutView.as_view(), name="logout"),
     path("prose/", include("prose.urls")),
     path("__reload__/", include("django_browser_reload.urls")),
-]
+] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

manuscript/admin.py

@@ -10,6 +10,7 @@
     EditorialStatus,
     Folio,
     Library,
+    LineCode,
     Location,
     LocationAlias,
     Reference,
@@ -118,6 +119,11 @@ class AuthorityFileInline(admin.TabularInline):
     extra = 1
 
 
+class LineCodeInline(admin.TabularInline):
+    model = LineCode
+    extra = 1
+
+
 # Custom admin models --------------------------------------------
 class SingleManuscriptAdmin(ImportExportModelAdmin):
     inlines = [
@@ -188,15 +194,18 @@ class CodexAdmin(admin.ModelAdmin):
 
 @admin.register(Location)
 class LocationAdmin(ImportExportModelAdmin):
-    inlines = [LocationAliasInline]
     list_display = (
+        "placename_id",
         "country",
         "description_html",
         "latitude",
         "longitude",
+        "toponym_type",
         "get_related_folios",
         "id",
     )
+    search_fields = ("country", "description")
+    inlines = [LocationAliasInline, LineCodeInline]
 
     def description_html(self, obj):
         return format_html(obj.description) if obj.description else ""
@@ -259,6 +268,12 @@ def get_changeform_initial_data(self, request):
         return initial
 
 
+class StanzaTranslatedAdmin(admin.ModelAdmin):
+    list_display = ("stanza", "stanza_text", "language")
+    search_fields = ("stanza", "stanza_text")
+    list_filter = ("language",)
+
+
 # Register to the admin interface.
 
 admin.site.register(Library, LibraryAdmin)
@@ -267,7 +282,8 @@ def get_changeform_initial_data(self, request):
 admin.site.register(SingleManuscript, SingleManuscriptAdmin)
 admin.site.register(Stanza, StanzaAdmin)
 admin.site.register(StanzaVariant, StanzaVariantAdmin)
-admin.site.register(StanzaTranslated)
+admin.site.register(StanzaTranslated, StanzaTranslatedAdmin)
+admin.site.register(LineCode)
 
 admin.site.site_header = "La Sfera Admin"
 admin.site.site_title = "La Sfera Admin Portal"

manuscript/management/commands/load_aliases.py

@@ -0,0 +1,129 @@
+# The following script reads the "Toponym Variants" sheet from the Excel file
+# and creates a new ToponymVariant object for each row. It also associates the
+# toponym variant with the corresponding toponym object. The field we want to write to
+# for these variants is "placename_from_mss" for each toponym object.
+
+# The script is similar to the load_toponyms.py script, but with some modifications
+# to accommodate the different fields and relationships.
+
+import logging
+
+import numpy as np
+import pandas as pd
+from django.core.management.base import BaseCommand
+from django.db import IntegrityError, transaction
+
+from manuscript.models import Location, LocationAlias
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help_text = "Load data from an Excel file. This reads information about the toponym variants and imports them."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--filepath", type=str, help="filepath of excel file to load"
+        )
+        parser.add_argument("--sheetname", type=str, help="name of sheet to load")
+
+    def handle_error(self, index, e, row, column_name, column_value):
+        logger.error(
+            "Error loading data at row %s, column '%s' with value '%s': %s - %s",
+            index + 1,
+            column_name,
+            column_value,
+            type(e),
+            e,
+        )
+        logger.debug("Row data: \n%s", row)
+
+    def process_field(self, row, field_name, index, is_bool=False):
+        try:
+            field_value = row.get(field_name)
+            if field_value is not None and isinstance(field_value, str):
+                field_value = field_value.strip()
+            return field_value
+        except Exception as e:
+            self.handle_error(index, e, row, field_name, row.get(field_name))
+            raise e
+
+    def create_location_alias(self, place_id, toponym):
+        try:
+            # Look up the Location instance using place_id
+            locations = Location.objects.filter(placename_id=place_id)
+
+            if not locations.exists():
+                logger.error("Location with placename_id %s does not exist", place_id)
+                return
+
+            for location in locations:
+                # Create or get the LocationAlias
+                location_alias, created = LocationAlias.objects.get_or_create(
+                    location=location,
+                    placename_from_mss=toponym,
+                )
+                if created:
+                    logger.info("Created new LocationAlias: %s", location_alias)
+                else:
+                    logger.info("LocationAlias already exists: %s", location_alias)
+        except IntegrityError as e:
+            logger.error("Integrity error creating LocationAlias: %s", e)
+        except IntegrityError as e:
+            logger.error("Integrity error creating LocationAlias: %s", e)
+        except Exception as e:
+            logger.error("Error creating LocationAlias: %s", e)
+
+    def handle(self, *args, **options):
+        filepath = options.get("filepath")
+        sheet_name = options.get("sheetname")
+
+        try:
+            with transaction.atomic():
+                self.load_data(filepath, sheet_name)
+                logger.info("Data loaded successfully")
+        except FileNotFoundError as e:
+            logger.error("Error loading data: %s.", e)
+
+    def load_data(self, filepath: str, sheet_name: str):
+        try:
+            logger.info("Loading data from %s sheet %s", filepath, sheet_name)
+            xls = pd.ExcelFile(filepath)
+
+            if sheet_name:
+                df = pd.read_excel(xls, sheet_name)
+                df = df.replace({np.nan: None})
+                df.columns = (
+                    df.columns.str.strip()
+                    .str.lower()
+                    .str.replace("[^\w\s]", "")
+                    .str.replace(" ", "_")
+                )
+                dfs = {sheet_name: df}
+            else:
+                dfs = pd.read_excel(xls, sheet_name=None)
+                for sheet_name, df in dfs.items():
+                    df = df.replace({np.nan: None})
+                    df.columns = (
+                        df.columns.str.strip()
+                        .str.lower()
+                        .str.replace("[^\w\s]", "")
+                        .str.replace(" ", "_")
+                    )
+                    dfs[sheet_name] = df
+
+            for sheet_name, df in dfs.items():
+                for index, row in df.iterrows():
+                    place_id = self.process_field(row, "place_id", index)
+                    toponym = self.process_field(row, "toponym", index)
+
+                    try:
+                        self.create_location_alias(place_id, toponym)
+                    except IntegrityError as e:
+                        logger.error("Integrity error creating LocationAlias: %s", e)
+                    except Exception as e:
+                        logger.error("Error creating LocationAlias: %s", e)
+
+        except Exception as e:
+            logger.error("Error loading data: %s", e)
+            raise e

manuscript/migrations/0086_merge_20240725_0846.py

@@ -0,0 +1,12 @@
+# Generated by Django 5.0.2 on 2024-07-25 12:46
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("manuscript", "0076_rename_group_editorialstatus_decorative_group_and_more"),
+        ("manuscript", "0085_remove_location_related_folio"),
+    ]
+
+    operations = []

manuscript/migrations/0087_singlemanuscript_photographs.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.2 on 2024-07-25 12:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("manuscript", "0086_merge_20240725_0846"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="singlemanuscript",
+            name="photographs",
+            field=models.FileField(
+                blank=True,
+                help_text="Upload photographs of the manuscript.",
+                null=True,
+                upload_to="",
+            ),
+        ),
+    ]

manuscript/migrations/0088_alter_singlemanuscript_siglum.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.2 on 2024-07-25 13:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("manuscript", "0087_singlemanuscript_photographs"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="singlemanuscript",
+            name="siglum",
+            field=models.CharField(
+                blank=True, db_index=True, max_length=20, null=True, unique=True
+            ),
+        ),
+    ]

manuscript/migrations/0089_location_code_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.2 on 2024-07-25 15:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("manuscript", "0088_alter_singlemanuscript_siglum"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="location",
+            name="code_id",
+            field=models.CharField(
+                blank=True, max_length=100, null=True, verbose_name="Code ID"
+            ),
+        ),
+    ]

manuscript/migrations/0090_linecode_remove_location_line_code_and_more.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.0.2 on 2024-07-25 15:34
+
+from django.db import migrations, models
+
+import manuscript.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("manuscript", "0089_location_code_id"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="LineCode",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "code",
+                    models.CharField(
+                        blank=True,
+                        help_text="Input the text by book, stanza, and line number. For example: 01.01.01 refers to book 1, stanza 1, line 1.",
+                        max_length=255,
+                        null=True,
+                        validators=[manuscript.models.validate_line_number_code],
+                    ),
+                ),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name="location",
+            name="line_code",
+        ),
+        migrations.AddField(
+            model_name="location",
+            name="line_code",
+            field=models.ManyToManyField(
+                blank=True,
+                help_text="Citation line codes where the toponym appears.",
+                to="manuscript.linecode",
+            ),
+        ),
+    ]

manuscript/migrations/0091_linecode_associated_iiif_url.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.2 on 2024-07-25 15:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("manuscript", "0090_linecode_remove_location_line_code_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="linecode",
+            name="associated_iiif_url",
+            field=models.URLField(
+                blank=True,
+                help_text="The URL to the IIIF manifest for the manuscript. If there isn't one, leave blank.",
+                max_length=255,
+                null=True,
+                verbose_name="Associated IIIF URL",
+            ),
+        ),
+    ]