Translation Alignments: Port translation alignment endpoints from Flask app to ATLAS

.gitignore

@@ -8,3 +8,4 @@ Pipfile
 Pipfile.lock
 .venv/
 .hypothesis/
+.coverage

README.md

@@ -181,6 +181,49 @@ level of `Version` nodes, maintaining the tree structure in the final payload.
 }
 ```
 
+## Text Alignments
+
+### Sample Queries
+
+Get text alignment chunks for a given reference:
+```
+{
+  textAlignmentChunks(reference: "urn:cts:greekLit:tlg0012.tlg001.perseus-grc2:1.8") {
+    edges {
+      cursor
+      node {
+        id
+        citation
+        items
+        alignment {
+          name
+        }
+      }
+    }
+  }
+}
+```
+
+Get a version annotated with text alignment chunks:
+```
+{
+  versions (urn:"urn:cts:greekLit:tlg0012.tlg001.perseus-grc2:") {
+    edges {
+      node {
+        metadata,
+        textAlignmentChunks (first:2){
+          edges {
+            node {
+              citation
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
 ## Tests
 
 Invoke tests via:

data/alignments/metadata.json

@@ -0,0 +1,20 @@
+{
+  "alignments": [
+    {
+      "version_urn": "urn:cts:greekLit:tlg0012.tlg001.perseus-grc2",
+      "content_path": "tlg0012.tlg001.perseus-grc2_aligned_to_tlg0012.tlg001.perseus-eng3.csv",
+      "metadata": {
+        "aligned_version_urn": "urn:cts:greekLit:tlg0012.tlg001.perseus-eng3",
+        "name": "Iliad (English)"
+      }
+    },
+    {
+      "version_urn": "urn:cts:greekLit:tlg0012.tlg002.perseus-grc2",
+      "content_path": "tlg0012.tlg002.perseus-grc2_aligned_to_tlg0012.tlg002.perseus-eng3.csv",
+      "metadata": {
+        "aligned_version_urn": "urn:cts:greekLit:tlg0012.tlg002.perseus-eng3",
+        "name": "Odyssey (English)"
+      }
+    }
+  ]
+}

readhomer_atlas/library/admin.py

@@ -3,11 +3,33 @@
 from treebeard.admin import TreeAdmin
 from treebeard.forms import movenodeform_factory
 
-from .models import Node
+from .models import Node, TextAlignment, TextAlignmentChunk
 
 
+@admin.register(Node)
 class NodeAdmin(TreeAdmin):
     form = movenodeform_factory(Node)
 
 
-admin.site.register(Node, NodeAdmin)
+@admin.register(TextAlignment)
+class TextAlignmentAdmin(admin.ModelAdmin):
+    list_display = ("id", "name", "slug", "metadata", "version")
+    list_filter = ("version",)
+    search_fields = ("name", "slug")
+    prepopulated_fields = {"slug": ["name"]}
+
+
+@admin.register(TextAlignmentChunk)
+class TextAlignmentChunkAdmin(admin.ModelAdmin):
+    list_display = (
+        "id",
+        "citation",
+        "metadata",
+        "idx",
+        "version",
+        "alignment",
+        "start",
+        "end",
+    )
+    list_filter = ("version", "alignment")
+    raw_id_fields = ("start", "end", "version")

readhomer_atlas/library/importers/__init__.py

@@ -0,0 +1,4 @@
+from . import alignments, versions
+
+
+__all__ = ["alignments", "versions"]

readhomer_atlas/library/importers/alignments.py

@@ -0,0 +1,189 @@
+import csv
+import json
+import os
+import re
+
+from django.conf import settings
+
+from ..models import Node, TextAlignment, TextAlignmentChunk
+
+
+ALIGNMENTS_DATA_PATH = os.path.join(settings.PROJECT_ROOT, "data", "alignments")
+ALIGNMENTS_METADATA_PATH = os.path.join(ALIGNMENTS_DATA_PATH, "metadata.json")
+
+LINE_KIND_UNKNOWN = None
+LINE_KIND_NEW = "new"
+LINE_KIND_CONTINUES = "continues"
+LINE_KIND_CONTINUATION = "continuation"
+
+CITATION_REFERENCE_REGEX = re.compile(r"\d+\.\d+")
+CONTENT_REFERENCE_REGEX = re.compile(r"\[\d+\.\d+\]")
+CONTENT_FOOTNOTE_REGEX = re.compile(r"\[\d+\]")
+
+
+def get_citation(greek_content):
+    """
+    Extracts citation from Greek content.
+
+    The `citation` field in the CSV contains inaccuracies,
+    so we'll build the citation field ourselves.
+    """
+    refs = CITATION_REFERENCE_REGEX.findall(greek_content)
+    start = refs[0]
+    end = refs[-1]
+    if start == end:
+        return start
+    else:
+        return "-".join([start, end])
+
+
+def transform_greek_content(greek_content):
+    """
+    Removes reference headings and footnote markers from
+    the milestone's Greek content.
+    """
+    greek_content = CONTENT_FOOTNOTE_REGEX.sub("", greek_content)
+    references = CONTENT_REFERENCE_REGEX.findall(greek_content)
+    tokens = [v for v in greek_content.split(" ") if v]
+    content = []
+    for pos, reference in enumerate(references):
+        ref_idx = tokens.index(reference)
+        try:
+            next_idx = tokens.index(references[pos + 1])
+        except IndexError:
+            subset = tokens[ref_idx:]
+        else:
+            subset = tokens[ref_idx:next_idx]
+        ref_label = CITATION_REFERENCE_REGEX.findall(subset.pop(0))[0]
+        content.append((ref_label, " ".join(subset)))
+    return content
+
+
+def map_leaves_to_milestones(leaves_to_milestones_lu, milestone_id, pos, leaves):
+    """
+    Updates the `LEAVES_TO_MILESTONES` index with each
+    reference within a given citation.
+    """
+    for reference, text in leaves:
+        leaves_to_milestones_lu.setdefault(reference, []).append(pos)
+
+
+def get_alignment_milestones(path):
+    LEAVES_TO_MILESTONES = {}
+    ALIGNMENTS = []
+
+    with open(path) as f:
+        reader = csv.reader(f)
+        # discard header row
+        next(reader)
+        for pos, row in enumerate(reader):
+            milestone_id = row[0]
+            greek_content = row[2]
+            # @@@ Can't use the citation field as-is due to discrepancies
+            # in the source material
+            # citation = row[1]
+            citation = get_citation(greek_content)
+            # @@@ `map_leaves_to_milestones` could return text extracted from leaves
+            # in a generator, but since the greek_content contains partial references.
+            # we end up needing to just extract it from the CSV data
+            cleaned_greek_content = transform_greek_content(greek_content)
+
+            # Map leaves to milestones
+            map_leaves_to_milestones(
+                LEAVES_TO_MILESTONES, milestone_id, pos, cleaned_greek_content
+            )
+
+            ALIGNMENTS.append(
+                {
+                    "id": milestone_id,
+                    "citation": citation,
+                    "greek_content": cleaned_greek_content,
+                    "english_content": [
+                        # @@@ continuation
+                        (citation, row[3], LINE_KIND_UNKNOWN)
+                    ],
+                }
+            )
+
+    for milestone_idx, alignment in enumerate(ALIGNMENTS):
+        for pos, entry in enumerate(alignment["greek_content"]):
+            offsets = LEAVES_TO_MILESTONES[entry[0]]
+            if [milestone_idx] == offsets:
+                alignment["greek_content"][pos] += (LINE_KIND_NEW,)
+            elif offsets[0] == milestone_idx:
+                alignment["greek_content"][pos] += (LINE_KIND_CONTINUES,)
+            else:
+                alignment["greek_content"][pos] += (LINE_KIND_CONTINUATION,)
+
+    return {"ALIGNMENTS": ALIGNMENTS, "LEAVES_TO_MILESTONES": LEAVES_TO_MILESTONES}
+
+
+def _alignment_chunk_obj(version, line_lookup, alignment, milestone, milestone_idx):
+    chunk_obj = TextAlignmentChunk(
+        citation=milestone["citation"],
+        idx=milestone_idx,
+        version=version,
+        alignment=alignment,
+        items=[milestone["greek_content"], milestone["english_content"]],
+    )
+    try:
+        start_ref, end_ref = milestone["citation"].split("-")
+    except ValueError:
+        start_ref = end_ref = milestone["citation"]
+    start_chapter, start_verse = [int(i) for i in start_ref.split(".")]
+    end_chapter, end_verse = [int(i) for i in end_ref.split(".")]
+
+    try:
+        chunk_obj.start = line_lookup[f"{start_chapter}.{start_verse}"]
+        chunk_obj.end = line_lookup[f"{end_chapter}.{end_verse}"]
+        return chunk_obj
+    except KeyError:
+        # greek version mismatch; could be others
+        print(
+            f"Skipping milestone due to line(s) not found.  [alignment.name={alignment.name} citation={milestone['citation']} milestone_id={milestone['id']}]"
+        )
+        return
+
+
+def _build_line_lookup(version):
+    lookup = {}
+    citation_scheme = version.metadata["citation_scheme"]
+    for line in version.get_descendants().filter(kind=citation_scheme[-1]):
+        lookup[line.ref] = line
+    return lookup
+
+
+def _import_alignment(data):
+    full_content_path = os.path.join(ALIGNMENTS_DATA_PATH, data["content_path"])
+
+    milestones = get_alignment_milestones(full_content_path)
+    # the version urns in data need a trailing colon
+    version_urn = f'{data["version_urn"]}:'
+    version = Node.objects.get(urn=version_urn)
+    alignment, _ = TextAlignment.objects.update_or_create(
+        version=version,
+        name=data["metadata"]["name"],
+        defaults=dict(metadata=data["metadata"]),
+    )
+
+    chunks_created = 0
+    line_lookup = _build_line_lookup(version)
+    to_create = []
+    for milestone_idx, milestone in enumerate(milestones["ALIGNMENTS"]):
+        chunk = _alignment_chunk_obj(
+            version, line_lookup, alignment, milestone, milestone_idx
+        )
+        if chunk:
+            to_create.append(chunk)
+            chunks_created += 1
+    created = len(TextAlignmentChunk.objects.bulk_create(to_create, batch_size=500))
+    assert created == chunks_created
+
+
+def import_alignments(reset=False):
+    if reset:
+        TextAlignment.objects.all().delete()
+
+    alignments_metadata = json.load(open(ALIGNMENTS_METADATA_PATH))
+    for alignment_data in alignments_metadata["alignments"]:
+        _import_alignment(alignment_data)

readhomer_atlas/library/importers.py → readhomer_atlas/library/importers/versions.py

@@ -11,8 +11,8 @@
 
 from readhomer_atlas import constants
 
-from .models import Node
-from .urn import URN
+from ..models import Node
+from ..urn import URN
 
 
 LIBRARY_DATA_PATH = os.path.join(settings.PROJECT_ROOT, "data", "library")
@@ -268,8 +268,9 @@ def get_first_value_for_language(values, lang):
     return next(iter(filter(lambda x: x["lang"] == lang, values)), None).get("value")
 
 
-def import_versions():
-    Node.objects.filter(kind="nid").delete()
+def import_versions(reset=True):
+    if reset:
+        Node.objects.filter(kind="nid").delete()
 
     library = resolve_library()
 

readhomer_atlas/library/management/commands/prepare_db.py

@@ -22,4 +22,7 @@ def handle(self, *args, **options):
         call_command("migrate")
 
         self.stdout.write("--[Loading versions]--")
-        importers.import_versions()
+        importers.versions.import_versions(reset=True)
+
+        self.stdout.write("--[Loading alignments]--")
+        importers.alignments.import_alignments(reset=True)