From 48c62f60e21e8326643f1040775390f132ae28a0 Mon Sep 17 00:00:00 2001
From: Hayden McCormick <haydenwmccormick@gmail.com>
Date: Thu, 3 Aug 2023 16:07:54 -0400
Subject: [PATCH 01/13] MMIF uploads now generate static /display URL for
 direct GET access

---
 app.py                   | 37 +++++++++++++++++++++++++++----------
 iiif_utils.py            |  4 ++--
 ocr.py                   | 32 +++++++++++++++++++++-----------
 templates/image.html     |  3 +++
 templates/ocr.html       |  5 +++--
 templates/pre-ocr.html   |  3 ++-
 templates/uv_player.html |  2 +-
 utils.py                 | 22 ++++++++++++----------
 8 files changed, 71 insertions(+), 37 deletions(-)

diff --git a/app.py b/app.py
index be37df7..0f11524 100644
--- a/app.py
+++ b/app.py
@@ -4,8 +4,9 @@
 import secrets
 import json
 import html
+import uuid
 
-from flask import request, render_template, flash, redirect, send_from_directory, session
+from flask import request, render_template, flash, redirect, send_from_directory, session, redirect
 from werkzeug.utils import secure_filename
 from mmif.serialize import Mmif
 
@@ -20,10 +21,10 @@ def index():
 def ocr():
     try:
         data = dict(request.json)
-        mmif_str = open(session["mmif_file"]).read()
+        mmif_str = open(os.path.join("/app", "static", data["mmif_id"], "file.mmif")).read()
         mmif = Mmif(mmif_str)
         ocr_view = mmif.get_view_by_id(data["view_id"])
-        return prepare_ocr_visualization(mmif, ocr_view)
+        return prepare_ocr_visualization(mmif, ocr_view, data["mmif_id"])
     except Exception as e:
         return f'<p class="error">{e}</h1>'
 
@@ -32,7 +33,7 @@ def ocr():
 def ocrpage():
     data = request.json
     try:
-        return (render_ocr(data['vid_path'], data["view_id"], data["page_number"]))
+        return (render_ocr(data["mmif_id"], data['vid_path'], data["view_id"], data["page_number"]))
     except Exception as e:
         return f'<p class="error">Unexpected error of type {type(e)}: {e}</h1>'
 
@@ -56,14 +57,30 @@ def upload():
             flash('WARNING: no file was selected')
             return redirect(request.url)
         if file:
-            filename = secure_filename(file.filename)
-            file.save(os.path.join('temp', filename))
-            with open("temp/" + filename) as fh:
-                session["mmif_file"] = fh.name
+            # filename = secure_filename(file.filename)
+            id = str(uuid.uuid4())
+            session["mmif_id"] = id
+            path = os.path.join("/app", "static", id)
+            os.makedirs(path)
+
+            file.save(os.path.join(path, "file.mmif"))
+            with open(os.path.join(path, "file.mmif")) as fh:
                 mmif_str = fh.read()
-                return render_mmif(mmif_str)
+            html_page = render_mmif(mmif_str)
+            file.save(os.path.join(path, "index.html"))
+            with open(os.path.join(path, "index.html"), "w") as f:
+                f.write(html_page)
+            return redirect(f"/display/{id}", code=302)
+        
     return render_template('upload.html')
 
+@app.route('/display/<id>')
+def display(id):
+    print ("THE ID IS " + id)
+    path = os.path.join("/app", "static", id)
+    with open(os.path.join(path, "index.html")) as f:
+        html_file = f.read()
+    return html_file
 
 @app.route('/uv/<path:path>')
 def send_js(path):
@@ -75,7 +92,7 @@ def render_mmif(mmif_str):
     media = get_media(mmif)
     annotations = prep_annotations(mmif)
     return render_template('player.html',
-                           mmif=mmif, media=media, annotations=annotations)
+                           media=media, annotations=annotations)
 
 
 if __name__ == '__main__':
diff --git a/iiif_utils.py b/iiif_utils.py
index 47a19b7..237f63c 100644
--- a/iiif_utils.py
+++ b/iiif_utils.py
@@ -5,7 +5,7 @@
 from typing import Dict
 
 import mmif
-from flask import url_for
+from flask import url_for, session
 from mmif import AnnotationTypes, DocumentTypes, Mmif
 import datetime
 
@@ -107,7 +107,7 @@ def add_structure_from_timeframe(in_mmif: Mmif, iiif_json: Dict):
 
 def save_manifest(iiif_json: Dict) -> str:
     # generate a iiif manifest and save output file
-    manifest = tempfile.NamedTemporaryFile('w', dir=str(pathlib.Path(__file__).parent /'static'/'tmp'), suffix='.json', delete=False)
+    manifest = tempfile.NamedTemporaryFile('w', dir=str(pathlib.Path(__file__).parent /'static'/session["mmif_id"]), suffix='.json', delete=False)
     json.dump(iiif_json, manifest, indent=4)
     return manifest.name
 
diff --git a/ocr.py b/ocr.py
index 2b88360..f6667e1 100644
--- a/ocr.py
+++ b/ocr.py
@@ -6,8 +6,9 @@
 import json
 import re
 import html
+import os, shutil
 
-from flask import render_template, session
+from flask import render_template
 from mmif.utils.video_document_helper import convert_timepoint, convert_timeframe
 
 
@@ -124,14 +125,15 @@ def paginate(frames_list):
 
     return {i: page for (i, page) in enumerate(pages)}
 
-def render_ocr(vid_path, view_id, page_number):
+def render_ocr(mmif_id, vid_path, view_id, page_number):
     """Iterate through frames and display the contents/alignments."""
     # Path for storing temporary images generated by cv2
     cv2_vid = cv2.VideoCapture(vid_path)
-    f = open(session[f"{view_id}-page-file"])
+    f = open(os.path.join("/app/static", mmif_id, f"{view_id}-pages.json"))
     frames_pages = json.load(f)
     page = frames_pages[str(page_number)]
     prev_frame_cap = None
+    path = make_image_directory(mmif_id)
     for frame_num, frame in page:
         # If index is range instead of frame...
         if frame.get("range"):
@@ -144,9 +146,8 @@ def render_ocr(vid_path, view_id, page_number):
         # Double check histogram similarity of "repeat" frames -- if they're significantly different, un-mark as repeat
         if prev_frame_cap is not None and frame["repeat"] and not is_duplicate_image(prev_frame_cap, frame_cap, cv2_vid):
             frame["repeat"] = False
-
         with tempfile.NamedTemporaryFile(
-                prefix=str(pathlib.Path(__file__).parent /'static'/'tmp'), suffix=".jpg", delete=False) as tf:
+                prefix=path, suffix=".jpg", delete=False) as tf:
             cv2.imwrite(tf.name, frame_cap)
             # "id" is just the name of the temp image file
             frame["id"] = pathlib.Path(tf.name).name
@@ -157,7 +158,17 @@ def render_ocr(vid_path, view_id, page_number):
                            view_id=view_id,
                            page=page,
                            n_pages=len(frames_pages),
-                           page_number=str(page_number))
+                           page_number=str(page_number),
+                           mmif_id = mmif_id)
+
+
+def make_image_directory(mmif_id):
+    # Make path for temp OCR image files or clear image files if it exists
+    path = os.path.join("/app", "static", mmif_id, "img/")
+    if os.path.exists(path):
+        shutil.rmtree(path)
+    os.makedirs(path)
+    return path
 
 
 def find_duplicates(frames_list, cv2_vid):
@@ -236,8 +247,7 @@ def get_ocr_views(mmif):
                 continue
     return views
 
-def save_json(dict, view_id):
-    with tempfile.NamedTemporaryFile(prefix=str(pathlib.Path(__file__).parent /'static'/'tmp'), suffix=".json", delete=False) as tf:
-        pages_json = open(tf.name, "w")
-        json.dump(dict, pages_json)
-        session[f"{view_id}-page-file"] = tf.name
+def save_json(dict, view_id, mmif_id):
+    path = os.path.join("/app/static", mmif_id, f"{view_id}-pages.json")
+    with open(path, 'w') as f:
+        json.dump(dict, f)
\ No newline at end of file
diff --git a/templates/image.html b/templates/image.html
index 1c07297..4aa066d 100644
--- a/templates/image.html
+++ b/templates/image.html
@@ -3,6 +3,8 @@
 
 <script>
 
+    console.log("{{filename}} is filename")
+
     drawImage();
 
     function drawImage() {
@@ -15,6 +17,7 @@
         imageObj.src = '{{filename}}';
 
         imageObj.onload = function() {
+            console.log("...loaded?");
             
             var imgWidth = imageObj.naturalWidth;
             var imgHeight = imageObj.naturalHeight;
diff --git a/templates/ocr.html b/templates/ocr.html
index 35b624d..03b451f 100644
--- a/templates/ocr.html
+++ b/templates/ocr.html
@@ -1,7 +1,7 @@
 
 <div id="ocr_tab_{{view_id}}">
     {% for frame_num, frame in page %}
-        {% set filename = frame["id"] %}
+        {% set filename = "/" + mmif_id + "/img/" + frame["id"] %}
         {% set id = frame["id"] %}
         {% set boxes = frame["boxes"] %}
         {% set secs = frame["secs"] %}
@@ -127,7 +127,8 @@ <h4>
         var data = {
             "vid_path": "{{vid_path}}",
             "view_id": "{{view_id}}",
-            "page_number": parseInt("{{page_number}}")
+            "page_number": parseInt("{{page_number}}"),
+            "mmif_id": "{{mmif_id}}"
         }
         if (page == BACKWARD) {
             data["page_number"] -= 1
diff --git a/templates/pre-ocr.html b/templates/pre-ocr.html
index d877c80..eba08d3 100644
--- a/templates/pre-ocr.html
+++ b/templates/pre-ocr.html
@@ -11,7 +11,8 @@
             return
         loaded_ocr_tabs += "{{tabname}}";
         var data = {
-            "view_id": "{{view_id}}"
+            "view_id": "{{view_id}}",
+            "mmif_id": "{{mmif_id}}"
         }
         $.ajax({
             type:'POST',
diff --git a/templates/uv_player.html b/templates/uv_player.html
index 908236d..ada4669 100644
--- a/templates/uv_player.html
+++ b/templates/uv_player.html
@@ -20,7 +20,7 @@
     $(".nav-item.UV").click(function() {
         if (!uvLoaded) {
             const data = {
-                manifest: "/tmp/{{manifest}}",
+                manifest: "/{{mmif_id}}/{{manifest}}",
                 embedded: true
              };
 
diff --git a/utils.py b/utils.py
index 1a7150a..ee5845a 100644
--- a/utils.py
+++ b/utils.py
@@ -7,7 +7,7 @@
 import displacy
 import tempfile
 
-from flask import Flask, render_template
+from flask import Flask, render_template, session
 from werkzeug.utils import secure_filename
 
 from mmif.serialize import Mmif, View
@@ -26,7 +26,7 @@
 app.secret_key = 'your_secret_key_here'
 
 def get_alignments(alignment_view):
-    vtt_file = tempfile.NamedTemporaryFile('w', dir="static/", suffix='.vtt', delete=False)
+    vtt_file = tempfile.NamedTemporaryFile('w', dir=f"/app/static/{session['mmif_id']}", suffix='.vtt', delete=False)
     vtt_file.write("WEBVTT\n\n")
     annotations = alignment_view.annotations
     timeframe_at_type = [at_type for at_type in alignment_view.metadata.contains if at_type.shortname == "TimeFrame" ][0]
@@ -94,7 +94,7 @@ def get_media(mmif):
         media.append((doc_type, document.id, doc_path, html))
     manifest_filename = generate_iiif_manifest(mmif)
     man = os.path.basename(manifest_filename)
-    temp = render_template("uv_player.html", manifest=man)
+    temp = render_template("uv_player.html", manifest=man, mmif_id=session["mmif_id"])
     media.append(('UV', "", "", temp))
     return media
 
@@ -154,9 +154,8 @@ def prep_annotations(mmif):
     for ocr_view in ocr_views:
         if not ocr_view.annotations:
             continue
-        # visualization = "prepare_ocr_visualization(mmif, ocr_view)"
         tabname = "Frames-%s" % ocr_view.id
-        visualization = render_template("pre-ocr.html", view_id=ocr_view.id, tabname=tabname)
+        visualization = render_template("pre-ocr.html", view_id=ocr_view.id, tabname=tabname, mmif_id=session["mmif_id"])
         tabs.append((tabname, visualization))
     return tabs
 
@@ -245,9 +244,11 @@ def html_video(vpath, vtt_srcview=None):
     html.write(f'    <source src=\"{vpath}\">\n')
     if vtt_srcview is not None:
         vtt_path = view_to_vtt(vtt_srcview)
+        src = "/" + os.sep.join(vtt_path.split(os.sep)[-2:])
         # use only basename because "static" directory is mapped to '' route by
         # `static_url_path` param
-        src = os.path.basename(vtt_path)
+        # src = os.path.basename(vtt_path)
+
         html.write(f'    <track kind="subtitles" srclang="en" src="{src}" label="English" default>\n')
     html.write("</video>\n")
     return html.getvalue()
@@ -326,7 +327,8 @@ def get_ner_views(mmif):
 def view_to_vtt(alignment_view):
     """Write alignments to a file in VTT style and return the filename."""
     vtt_file = get_alignments(alignment_view)
-    return os.sep.join(vtt_file.name.split(os.sep)[-2:])
+    return vtt_file.name
+    # return os.sep.join(vtt_file.name.split(os.sep)[-3:])
 
 def create_ner_visualization(mmif, view):
     metadata = view.metadata.contains.get(Uri.NE)
@@ -354,7 +356,7 @@ def get_properties(annotation):
 
 # OCR Tools ----------------------
 
-def prepare_ocr_visualization(mmif, view):
+def prepare_ocr_visualization(mmif, view, mmif_id):
     """ Visualize OCR by extracting image frames with BoundingBoxes from video"""
     # frames, text_docs, alignments = {}, {}, {}
     vid_path = get_video_path(mmif)
@@ -368,5 +370,5 @@ def prepare_ocr_visualization(mmif, view):
     frames_list = find_duplicates(frames_list, cv2_vid)
     frames_pages = paginate(frames_list)
     # Save page list as temp file
-    save_json(frames_pages, view.id)
-    return render_ocr(vid_path, view.id, 0)
\ No newline at end of file
+    save_json(frames_pages, view.id, mmif_id)
+    return render_ocr(mmif_id, vid_path, view.id, 0)
\ No newline at end of file

From 4c93a13ab2cb8738ac7daef8c54503aaf8db97c7 Mon Sep 17 00:00:00 2001
From: Hayden McCormick <haydenwmccormick@gmail.com>
Date: Fri, 4 Aug 2023 15:14:47 -0400
Subject: [PATCH 02/13] Implement cache thread for automatic deletion of old
 tmp files

---
 app.py                   | 73 +++++++++++++++++++++++++++++++++++-----
 iiif_utils.py            |  2 +-
 ocr.py                   |  7 ++--
 requirements.txt         |  1 +
 templates/image.html     |  3 --
 templates/ocr.html       |  2 +-
 templates/uv_player.html |  2 +-
 utils.py                 |  3 +-
 8 files changed, 72 insertions(+), 21 deletions(-)

diff --git a/app.py b/app.py
index 0f11524..56211b8 100644
--- a/app.py
+++ b/app.py
@@ -2,9 +2,10 @@
 import pathlib
 import sys
 import secrets
-import json
-import html
 import uuid
+import threading
+import shutil
+import time
 
 from flask import request, render_template, flash, redirect, send_from_directory, session, redirect
 from werkzeug.utils import secure_filename
@@ -12,6 +13,9 @@
 
 from utils import app, render_ocr, get_media, prep_annotations, prepare_ocr_visualization
 
+global cleanup_thread
+cleanup_thread = None
+
 @app.route('/')
 def index():
     return render_template('index.html')
@@ -21,7 +25,7 @@ def index():
 def ocr():
     try:
         data = dict(request.json)
-        mmif_str = open(os.path.join("/app", "static", data["mmif_id"], "file.mmif")).read()
+        mmif_str = open(os.path.join("/app/static/tmp", data["mmif_id"], "file.mmif")).read()
         mmif = Mmif(mmif_str)
         ocr_view = mmif.get_view_by_id(data["view_id"])
         return prepare_ocr_visualization(mmif, ocr_view, data["mmif_id"])
@@ -57,17 +61,16 @@ def upload():
             flash('WARNING: no file was selected')
             return redirect(request.url)
         if file:
-            # filename = secure_filename(file.filename)
             id = str(uuid.uuid4())
             session["mmif_id"] = id
-            path = os.path.join("/app", "static", id)
+            path = os.path.join("/app/static/tmp", id)
             os.makedirs(path)
 
+            set_last_access(path)
             file.save(os.path.join(path, "file.mmif"))
             with open(os.path.join(path, "file.mmif")) as fh:
                 mmif_str = fh.read()
             html_page = render_mmif(mmif_str)
-            file.save(os.path.join(path, "index.html"))
             with open(os.path.join(path, "index.html"), "w") as f:
                 f.write(html_page)
             return redirect(f"/display/{id}", code=302)
@@ -76,8 +79,8 @@ def upload():
 
 @app.route('/display/<id>')
 def display(id):
-    print ("THE ID IS " + id)
-    path = os.path.join("/app", "static", id)
+    path = os.path.join("/app/static/tmp", id)
+    set_last_access(path)
     with open(os.path.join(path, "index.html")) as f:
         html_file = f.read()
     return html_file
@@ -86,6 +89,9 @@ def display(id):
 def send_js(path):
     return send_from_directory("uv", path)
 
+def set_last_access(path):
+    with open(os.path.join(path, "last_access.txt"), "w") as f:
+        f.write(str(time.time()))
 
 def render_mmif(mmif_str):
     mmif = Mmif(mmif_str)
@@ -94,6 +100,49 @@ def render_mmif(mmif_str):
     return render_template('player.html',
                            media=media, annotations=annotations)
 
+def scan_tmp_directory():
+    total_size = 0
+    oldest_accessed_dir = {"dir": None, "access_time": None}
+    for path, dirs, files in os.walk("/app/static/tmp"):
+        for f in files:
+            fp = os.path.join(path, f)
+            total_size += os.path.getsize(fp)
+
+        # Get oldest accessed directory for deletion
+        if os.path.dirname(path) != "/app/static/tmp":
+            continue
+        if not os.path.isfile(os.path.join(path, "last_access.txt")):
+            oldest_accessed_dir = {"dir": path, "access_time": 0}
+        elif oldest_accessed_dir["dir"] is None:
+            with open(os.path.join(path, "last_access.txt")) as f:
+                timestamp = f.read()
+                if timestamp == '':
+                    continue
+                oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
+        else:
+            with open(os.path.join(path, "last_access.txt")) as f:
+                if float(f.read()) < oldest_accessed_dir["access_time"]:
+                    timestamp = f.read()
+                    if timestamp == '':
+                        continue
+                    oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
+    
+    return total_size, oldest_accessed_dir["dir"]
+
+
+def cleanup():
+    print("Checking visualization cache...")
+    # Max tmp size is 50MB
+    max_size = 10000000
+    folder_size, oldest_dir = scan_tmp_directory()
+    while folder_size > max_size:
+        print(f"Maximum cache size reached. Deleting {os.path.basename(oldest_dir)}.")
+        shutil.rmtree(oldest_dir)
+        folder_size, oldest_dir = scan_tmp_directory()
+    # Cleanup again in 12 hours
+    time.sleep(43200)
+    cleanup()
+
 
 if __name__ == '__main__':
     # Make path for temp files
@@ -109,4 +158,10 @@ def render_mmif(mmif_str):
     port = 5000
     if len(sys.argv) > 2 and sys.argv[1] == '-p':
         port = int(sys.argv[2])
-    app.run(port=port, host='0.0.0.0', debug=True)
+
+    if cleanup_thread is None:
+        cleanup_thread = threading.Timer(5, cleanup)
+        cleanup_thread.daemon = True
+        cleanup_thread.start()
+        
+    app.run(port=port, host='0.0.0.0', debug=True, use_reloader=False)
diff --git a/iiif_utils.py b/iiif_utils.py
index 237f63c..5ecad94 100644
--- a/iiif_utils.py
+++ b/iiif_utils.py
@@ -107,7 +107,7 @@ def add_structure_from_timeframe(in_mmif: Mmif, iiif_json: Dict):
 
 def save_manifest(iiif_json: Dict) -> str:
     # generate a iiif manifest and save output file
-    manifest = tempfile.NamedTemporaryFile('w', dir=str(pathlib.Path(__file__).parent /'static'/session["mmif_id"]), suffix='.json', delete=False)
+    manifest = tempfile.NamedTemporaryFile('w', dir=str(pathlib.Path(__file__).parent /'static/tmp'/session["mmif_id"]), suffix='.json', delete=False)
     json.dump(iiif_json, manifest, indent=4)
     return manifest.name
 
diff --git a/ocr.py b/ocr.py
index f6667e1..202ea00 100644
--- a/ocr.py
+++ b/ocr.py
@@ -5,7 +5,6 @@
 import tempfile
 import json
 import re
-import html
 import os, shutil
 
 from flask import render_template
@@ -129,7 +128,7 @@ def render_ocr(mmif_id, vid_path, view_id, page_number):
     """Iterate through frames and display the contents/alignments."""
     # Path for storing temporary images generated by cv2
     cv2_vid = cv2.VideoCapture(vid_path)
-    f = open(os.path.join("/app/static", mmif_id, f"{view_id}-pages.json"))
+    f = open(os.path.join("/app/static/tmp", mmif_id, f"{view_id}-pages.json"))
     frames_pages = json.load(f)
     page = frames_pages[str(page_number)]
     prev_frame_cap = None
@@ -164,7 +163,7 @@ def render_ocr(mmif_id, vid_path, view_id, page_number):
 
 def make_image_directory(mmif_id):
     # Make path for temp OCR image files or clear image files if it exists
-    path = os.path.join("/app", "static", mmif_id, "img/")
+    path = os.path.join("/app/static/tmp", mmif_id, "img/")
     if os.path.exists(path):
         shutil.rmtree(path)
     os.makedirs(path)
@@ -248,6 +247,6 @@ def get_ocr_views(mmif):
     return views
 
 def save_json(dict, view_id, mmif_id):
-    path = os.path.join("/app/static", mmif_id, f"{view_id}-pages.json")
+    path = os.path.join("/app/static/tmp", mmif_id, f"{view_id}-pages.json")
     with open(path, 'w') as f:
         json.dump(dict, f)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 563435d..17f9b7b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,5 @@ spacy==2.3.2
 mmif-python==1.0.8
 mmif-python[utils]==1.0.8
 flask-session
+flask[async]
 opencv-python==4.*
\ No newline at end of file
diff --git a/templates/image.html b/templates/image.html
index 4aa066d..1c07297 100644
--- a/templates/image.html
+++ b/templates/image.html
@@ -3,8 +3,6 @@
 
 <script>
 
-    console.log("{{filename}} is filename")
-
     drawImage();
 
     function drawImage() {
@@ -17,7 +15,6 @@
         imageObj.src = '{{filename}}';
 
         imageObj.onload = function() {
-            console.log("...loaded?");
             
             var imgWidth = imageObj.naturalWidth;
             var imgHeight = imageObj.naturalHeight;
diff --git a/templates/ocr.html b/templates/ocr.html
index 03b451f..08916c8 100644
--- a/templates/ocr.html
+++ b/templates/ocr.html
@@ -1,7 +1,7 @@
 
 <div id="ocr_tab_{{view_id}}">
     {% for frame_num, frame in page %}
-        {% set filename = "/" + mmif_id + "/img/" + frame["id"] %}
+        {% set filename = "/tmp/" + mmif_id + "/img/" + frame["id"] %}
         {% set id = frame["id"] %}
         {% set boxes = frame["boxes"] %}
         {% set secs = frame["secs"] %}
diff --git a/templates/uv_player.html b/templates/uv_player.html
index ada4669..b6d0fa1 100644
--- a/templates/uv_player.html
+++ b/templates/uv_player.html
@@ -20,7 +20,7 @@
     $(".nav-item.UV").click(function() {
         if (!uvLoaded) {
             const data = {
-                manifest: "/{{mmif_id}}/{{manifest}}",
+                manifest: "/tmp/{{mmif_id}}/{{manifest}}",
                 embedded: true
              };
 
diff --git a/utils.py b/utils.py
index ee5845a..2c3536c 100644
--- a/utils.py
+++ b/utils.py
@@ -26,7 +26,7 @@
 app.secret_key = 'your_secret_key_here'
 
 def get_alignments(alignment_view):
-    vtt_file = tempfile.NamedTemporaryFile('w', dir=f"/app/static/{session['mmif_id']}", suffix='.vtt', delete=False)
+    vtt_file = tempfile.NamedTemporaryFile('w', dir=f"/app/static/tmp/{session['mmif_id']}", suffix='.vtt', delete=False)
     vtt_file.write("WEBVTT\n\n")
     annotations = alignment_view.annotations
     timeframe_at_type = [at_type for at_type in alignment_view.metadata.contains if at_type.shortname == "TimeFrame" ][0]
@@ -275,7 +275,6 @@ def html_text(tpath):
 def html_img(ipath, boxes=None, id="imgCanvas"):
     ipath = url2posix(ipath)
     boxes = [] if boxes is None else boxes
-    # t = Template(open('templates/image.html').read())
     return render_template('image.html', filename=ipath, boxes=boxes, id=id)
 
 

From 5040166bcc4b68b082c5318a09e1889ad7ad273a Mon Sep 17 00:00:00 2001
From: Hayden McCormick <haydenwmccormick@gmail.com>
Date: Fri, 4 Aug 2023 15:24:34 -0400
Subject: [PATCH 03/13] Better compartmentalization and deletion handling

---
 app.py   | 65 +++++++++-----------------------------------------------
 cache.py | 50 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 55 deletions(-)
 create mode 100644 cache.py

diff --git a/app.py b/app.py
index 56211b8..6b3d2df 100644
--- a/app.py
+++ b/app.py
@@ -4,14 +4,13 @@
 import secrets
 import uuid
 import threading
-import shutil
-import time
 
 from flask import request, render_template, flash, redirect, send_from_directory, session, redirect
 from werkzeug.utils import secure_filename
 from mmif.serialize import Mmif
 
 from utils import app, render_ocr, get_media, prep_annotations, prepare_ocr_visualization
+from cache import set_last_access, scan_tmp_directory, cleanup
 
 global cleanup_thread
 cleanup_thread = None
@@ -79,20 +78,20 @@ def upload():
 
 @app.route('/display/<id>')
 def display(id):
-    path = os.path.join("/app/static/tmp", id)
-    set_last_access(path)
-    with open(os.path.join(path, "index.html")) as f:
-        html_file = f.read()
-    return html_file
+    try:
+        path = os.path.join("/app/static/tmp", id)
+        set_last_access(path)
+        with open(os.path.join(path, "index.html")) as f:
+            html_file = f.read()
+        return html_file
+    except FileNotFoundError:
+        flash("File not found -- please upload again (it may have been deleted to clear up cache space).")
+        return redirect("/upload")
 
 @app.route('/uv/<path:path>')
 def send_js(path):
     return send_from_directory("uv", path)
 
-def set_last_access(path):
-    with open(os.path.join(path, "last_access.txt"), "w") as f:
-        f.write(str(time.time()))
-
 def render_mmif(mmif_str):
     mmif = Mmif(mmif_str)
     media = get_media(mmif)
@@ -100,50 +99,6 @@ def render_mmif(mmif_str):
     return render_template('player.html',
                            media=media, annotations=annotations)
 
-def scan_tmp_directory():
-    total_size = 0
-    oldest_accessed_dir = {"dir": None, "access_time": None}
-    for path, dirs, files in os.walk("/app/static/tmp"):
-        for f in files:
-            fp = os.path.join(path, f)
-            total_size += os.path.getsize(fp)
-
-        # Get oldest accessed directory for deletion
-        if os.path.dirname(path) != "/app/static/tmp":
-            continue
-        if not os.path.isfile(os.path.join(path, "last_access.txt")):
-            oldest_accessed_dir = {"dir": path, "access_time": 0}
-        elif oldest_accessed_dir["dir"] is None:
-            with open(os.path.join(path, "last_access.txt")) as f:
-                timestamp = f.read()
-                if timestamp == '':
-                    continue
-                oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
-        else:
-            with open(os.path.join(path, "last_access.txt")) as f:
-                if float(f.read()) < oldest_accessed_dir["access_time"]:
-                    timestamp = f.read()
-                    if timestamp == '':
-                        continue
-                    oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
-    
-    return total_size, oldest_accessed_dir["dir"]
-
-
-def cleanup():
-    print("Checking visualization cache...")
-    # Max tmp size is 50MB
-    max_size = 10000000
-    folder_size, oldest_dir = scan_tmp_directory()
-    while folder_size > max_size:
-        print(f"Maximum cache size reached. Deleting {os.path.basename(oldest_dir)}.")
-        shutil.rmtree(oldest_dir)
-        folder_size, oldest_dir = scan_tmp_directory()
-    # Cleanup again in 12 hours
-    time.sleep(43200)
-    cleanup()
-
-
 if __name__ == '__main__':
     # Make path for temp files
     tmp_path = pathlib.Path(__file__).parent /'static'/'tmp'
diff --git a/cache.py b/cache.py
new file mode 100644
index 0000000..133d896
--- /dev/null
+++ b/cache.py
@@ -0,0 +1,50 @@
+import os
+import time
+import shutil
+
+def set_last_access(path):
+    with open(os.path.join(path, "last_access.txt"), "w") as f:
+        f.write(str(time.time()))
+
+def scan_tmp_directory():
+    total_size = 0
+    oldest_accessed_dir = {"dir": None, "access_time": None}
+    for path, dirs, files in os.walk("/app/static/tmp"):
+        for f in files:
+            fp = os.path.join(path, f)
+            total_size += os.path.getsize(fp)
+
+        # Get oldest accessed directory for deletion
+        if os.path.dirname(path) != "/app/static/tmp":
+            continue
+        if not os.path.isfile(os.path.join(path, "last_access.txt")):
+            oldest_accessed_dir = {"dir": path, "access_time": 0}
+        elif oldest_accessed_dir["dir"] is None:
+            with open(os.path.join(path, "last_access.txt")) as f:
+                timestamp = f.read()
+                if timestamp == '':
+                    continue
+                oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
+        else:
+            with open(os.path.join(path, "last_access.txt")) as f:
+                if float(f.read()) < oldest_accessed_dir["access_time"]:
+                    timestamp = f.read()
+                    if timestamp == '':
+                        continue
+                    oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
+    
+    return total_size, oldest_accessed_dir["dir"]
+
+
+def cleanup():
+    print("Checking visualization cache...")
+    # Max tmp size is 50MB
+    max_size = 10000000
+    folder_size, oldest_dir = scan_tmp_directory()
+    while folder_size > max_size:
+        print(f"Maximum cache size reached. Deleting {os.path.basename(oldest_dir)}.")
+        shutil.rmtree(oldest_dir)
+        folder_size, oldest_dir = scan_tmp_directory()
+    # Cleanup again in 12 hours
+    time.sleep(43200)
+    cleanup()

From 8fbb64c847ca10fb9f615fe3b87365f8a51f12ef Mon Sep 17 00:00:00 2001
From: Hayden McCormick <haydenwmccormick@gmail.com>
Date: Wed, 9 Aug 2023 12:01:24 -0400
Subject: [PATCH 04/13] Only check cache on upload, add option to return to
 upload page

---
 app.py                | 18 ++++++++++++------
 cache.py              | 21 +++++++++++----------
 templates/player.html | 25 ++++++++++++++++++++++++-
 3 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/app.py b/app.py
index 6b3d2df..f3fdec8 100644
--- a/app.py
+++ b/app.py
@@ -3,7 +3,7 @@
 import sys
 import secrets
 import uuid
-import threading
+from threading import Thread
 
 from flask import request, render_template, flash, redirect, send_from_directory, session, redirect
 from werkzeug.utils import secure_filename
@@ -60,11 +60,11 @@ def upload():
             flash('WARNING: no file was selected')
             return redirect(request.url)
         if file:
+            # Save file locally
             id = str(uuid.uuid4())
             session["mmif_id"] = id
             path = os.path.join("/app/static/tmp", id)
             os.makedirs(path)
-
             set_last_access(path)
             file.save(os.path.join(path, "file.mmif"))
             with open(os.path.join(path, "file.mmif")) as fh:
@@ -72,6 +72,12 @@ def upload():
             html_page = render_mmif(mmif_str)
             with open(os.path.join(path, "index.html"), "w") as f:
                 f.write(html_page)
+
+            # Perform cleanup
+            t = Thread(target=cleanup)
+            t.daemon = True
+            t.run()
+
             return redirect(f"/display/{id}", code=302)
         
     return render_template('upload.html')
@@ -114,9 +120,9 @@ def render_mmif(mmif_str):
     if len(sys.argv) > 2 and sys.argv[1] == '-p':
         port = int(sys.argv[2])
 
-    if cleanup_thread is None:
-        cleanup_thread = threading.Timer(5, cleanup)
-        cleanup_thread.daemon = True
-        cleanup_thread.start()
+    # if cleanup_thread is None:
+    #     cleanup_thread = threading.Timer(5, cleanup)
+    #     cleanup_thread.daemon = True
+    #     cleanup_thread.start()
         
     app.run(port=port, host='0.0.0.0', debug=True, use_reloader=False)
diff --git a/cache.py b/cache.py
index 133d896..5df45b8 100644
--- a/cache.py
+++ b/cache.py
@@ -1,6 +1,9 @@
 import os
 import time
 import shutil
+import threading
+
+lock = threading.Lock()
 
 def set_last_access(path):
     with open(os.path.join(path, "last_access.txt"), "w") as f:
@@ -37,14 +40,12 @@ def scan_tmp_directory():
 
 
 def cleanup():
-    print("Checking visualization cache...")
-    # Max tmp size is 50MB
-    max_size = 10000000
-    folder_size, oldest_dir = scan_tmp_directory()
-    while folder_size > max_size:
-        print(f"Maximum cache size reached. Deleting {os.path.basename(oldest_dir)}.")
-        shutil.rmtree(oldest_dir)
+    with lock:
+        print("Checking visualization cache...")
+        # Max tmp size is 50MB
+        max_size = 50000000
         folder_size, oldest_dir = scan_tmp_directory()
-    # Cleanup again in 12 hours
-    time.sleep(43200)
-    cleanup()
+        while folder_size > max_size:
+            print(f"Maximum cache size reached. Deleting {os.path.basename(oldest_dir)}.")
+            shutil.rmtree(oldest_dir)
+            folder_size, oldest_dir = scan_tmp_directory()
diff --git a/templates/player.html b/templates/player.html
index 1116688..930b085 100644
--- a/templates/player.html
+++ b/templates/player.html
@@ -23,6 +23,13 @@
 </head>
 
 <style>
+    .card-header {
+      display: flex;
+      flex-direction: row;
+      width: 100%;
+      justify-content: space-between;
+    }
+
     .tab-content {
         display: inline-block;
         overflow-y: auto;
@@ -62,6 +69,16 @@
         animation: spin 2s linear infinite;
     }
 
+    .left, .right, .title {
+      flex: 1;
+      display: flex;
+      align-items: center;
+    }
+
+    .title {
+      justify-content: center;
+    }
+    
     /* Safari */
     @-webkit-keyframes spin {
         0% { -webkit-transform: rotate(0deg); }
@@ -79,7 +96,13 @@
 <div class="panel panel-default">
 
   <div class="card-header">
-    <h1 align="center">Visualizing MMIF</h1>
+    <div class="left">
+      <form action="/upload">
+        <input type="submit" value="Upload another file" />
+      </form>
+    </div>
+    <h1 class="title">Visualizing MMIF</h1>
+    <div class="right"></div>
   </div>
 
   <div class="card-body container-fluid">

From e8bbefde6adfcfcc3504d9ddf99cf8468213e44e Mon Sep 17 00:00:00 2001
From: Hayden McCormick <haydenwmccormick@gmail.com>
Date: Wed, 9 Aug 2023 13:19:48 -0400
Subject: [PATCH 05/13] Update README + short UUIDs for URLs

---
 README.md        | 13 +++++++++--
 app.py           | 59 ++++++++++++++++++++++++------------------------
 requirements.txt |  3 ++-
 3 files changed, 42 insertions(+), 33 deletions(-)

diff --git a/README.md b/README.md
index 5304dce..c20026b 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ In these notes we assume that the data are in a local directory named `/Users/Sh
 $ docker run --rm -d -p 5000:5000 -v /Users/Shared/archive:/data clams-mmif-visualizer
 ```
 
-After this, all you need to do is point your browser at [http://0.0.0.0:5000/upload](http://0.0.0.0:5000/upload), click "Choose File", select a MMIF file and then click "Visualize". See the *Data source repository and input MMIF file* section below for a description of the MMIF file. Assuming you have not made any changes to the directory structure you can use the example MMIF files in the `input` folder.
+See the *Data source repository and input MMIF file* section below for a description of the MMIF file. Assuming you have not made any changes to the directory structure you can use the example MMIF files in the `input` folder.
 
 **Some background**
 
@@ -89,8 +89,17 @@ To run the server do:
 $ python app.py
 ```
 
-Then point your browser at [http://0.0.0.0:5000/upload](http://0.0.0.0:5000/upload), click "Choose File" and then click "Visualize".
 
+## Uploading Files
+MMIF files can be uploaded to the visualization server one of two ways:
+* Point your browser to http://0.0.0.0:5000/upload, click "Choose File" and then click "Visualize". This will generate a static URL containing the visualization of the input file (e.g. `http://localhost:5000/display/HaTxbhDfwakewakmzdXu5e`). Once the file is uploaded, the page will automatically redirect to the file's visualization.
+* Using a command line, enter:
+  ``` 
+  curl -X POST -F "file=@<filename>" -s http://localhost:5000/upload
+  ```
+  This will upload the file and print the unique identifier for the file visualization. The visualization can be accessed at `http://localhost:5000/display/<id>`
+
+The server will maintain a cache of up to 50MB for these temporary files, so the visualizations can be repeatedly accessed without needing to re-upload any files. Once this limit is reached, the server will delete stored visualizations until enough space is reclaimed, drawing from oldest/least recently accessed pages first. If you attempt to access the /display URL of a deleted file, you will be redirected back to the upload page instead.
 
 
 ## Data source repository and input MMIF file
diff --git a/app.py b/app.py
index f3fdec8..38bbc9d 100644
--- a/app.py
+++ b/app.py
@@ -2,7 +2,7 @@
 import pathlib
 import sys
 import secrets
-import uuid
+import shortuuid
 from threading import Thread
 
 from flask import request, render_template, flash, redirect, send_from_directory, session, redirect
@@ -10,10 +10,7 @@
 from mmif.serialize import Mmif
 
 from utils import app, render_ocr, get_media, prep_annotations, prepare_ocr_visualization
-from cache import set_last_access, scan_tmp_directory, cleanup
-
-global cleanup_thread
-cleanup_thread = None
+from cache import set_last_access, cleanup
 
 @app.route('/')
 def index():
@@ -48,7 +45,7 @@ def upload():
     if request.method == 'POST':
         # Check if request is coming from elasticsearch
         if 'data' in request.form:
-            return render_mmif(request.form['data'])
+            return upload_file(request.form['data'])
         # Otherwise, check if the post request has the file part
         elif 'file' not in request.files:
             flash('WARNING: post request has no file part')
@@ -60,25 +57,7 @@ def upload():
             flash('WARNING: no file was selected')
             return redirect(request.url)
         if file:
-            # Save file locally
-            id = str(uuid.uuid4())
-            session["mmif_id"] = id
-            path = os.path.join("/app/static/tmp", id)
-            os.makedirs(path)
-            set_last_access(path)
-            file.save(os.path.join(path, "file.mmif"))
-            with open(os.path.join(path, "file.mmif")) as fh:
-                mmif_str = fh.read()
-            html_page = render_mmif(mmif_str)
-            with open(os.path.join(path, "index.html"), "w") as f:
-                f.write(html_page)
-
-            # Perform cleanup
-            t = Thread(target=cleanup)
-            t.daemon = True
-            t.run()
-
-            return redirect(f"/display/{id}", code=302)
+            return upload_file(file)
         
     return render_template('upload.html')
 
@@ -105,6 +84,31 @@ def render_mmif(mmif_str):
     return render_template('player.html',
                            media=media, annotations=annotations)
 
+def upload_file(file):
+    # Save file locally
+    id = shortuuid.uuid()
+    session["mmif_id"] = id
+    path = os.path.join("/app/static/tmp", id)
+    os.makedirs(path)
+    set_last_access(path)
+    file.save(os.path.join(path, "file.mmif"))
+    with open(os.path.join(path, "file.mmif")) as fh:
+        mmif_str = fh.read()
+    html_page = render_mmif(mmif_str)
+    with open(os.path.join(path, "index.html"), "w") as f:
+        f.write(html_page)
+
+    # Perform cleanup
+    t = Thread(target=cleanup)
+    t.daemon = True
+    t.run()
+
+    agent = request.headers.get('User-Agent')
+    if 'curl' in agent.lower():
+        return f"Document ID is {id}\nYou can access the visualized file at /display/{id}\n"
+    return redirect(f"/display/{id}", code=302)
+
+
 if __name__ == '__main__':
     # Make path for temp files
     tmp_path = pathlib.Path(__file__).parent /'static'/'tmp'
@@ -119,10 +123,5 @@ def render_mmif(mmif_str):
     port = 5000
     if len(sys.argv) > 2 and sys.argv[1] == '-p':
         port = int(sys.argv[2])
-
-    # if cleanup_thread is None:
-    #     cleanup_thread = threading.Timer(5, cleanup)
-    #     cleanup_thread.daemon = True
-    #     cleanup_thread.start()
         
     app.run(port=port, host='0.0.0.0', debug=True, use_reloader=False)
diff --git a/requirements.txt b/requirements.txt
index 17f9b7b..ba0bfb1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,4 +3,5 @@ mmif-python==1.0.8
 mmif-python[utils]==1.0.8
 flask-session
 flask[async]
-opencv-python==4.*
\ No newline at end of file
+opencv-python==4.*
+shortuuid==1.0.11
\ No newline at end of file

From 7e69deef21fe3895c201c175f6e6cbe7e741d3b7 Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Sun, 27 Aug 2023 12:11:34 -0400
Subject: [PATCH 06/13] updated req.txt, increased cache size, added cache path
 getter helper

---
 cache.py         | 36 ++++++++++++++++++------------------
 requirements.txt |  4 ++--
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/cache.py b/cache.py
index 5df45b8..bcb84cf 100644
--- a/cache.py
+++ b/cache.py
@@ -2,48 +2,48 @@
 import time
 import shutil
 import threading
+import pathlib
 
 lock = threading.Lock()
 
+
+def get_cache_path():
+    return pathlib.Path(__file__).parent.parent / "static" / "tmp"
+
+
 def set_last_access(path):
     with open(os.path.join(path, "last_access.txt"), "w") as f:
         f.write(str(time.time()))
 
+
 def scan_tmp_directory():
-    total_size = 0
     oldest_accessed_dir = {"dir": None, "access_time": None}
-    for path, dirs, files in os.walk("/app/static/tmp"):
-        for f in files:
-            fp = os.path.join(path, f)
-            total_size += os.path.getsize(fp)
-
-        # Get oldest accessed directory for deletion
-        if os.path.dirname(path) != "/app/static/tmp":
-            continue
-        if not os.path.isfile(os.path.join(path, "last_access.txt")):
-            oldest_accessed_dir = {"dir": path, "access_time": 0}
+    total_size = sum(f.stat().st_size for f in get_cache_path().glob('**/*') if f.is_file())
+    # this will be some visualization IDs
+    for p in get_cache_path().glob('*'):
+        if not (p / 'last_access.txt').exists():
+            oldest_accessed_dir = {"dir": p, "access_time": 0}
         elif oldest_accessed_dir["dir"] is None:
-            with open(os.path.join(path, "last_access.txt")) as f:
+            with open(p / 'last_access.txt') as f:
                 timestamp = f.read()
                 if timestamp == '':
                     continue
-                oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
+                oldest_accessed_dir = {"dir": p, "access_time": float(timestamp)}
         else:
-            with open(os.path.join(path, "last_access.txt")) as f:
+            with open(p / 'last_access.txt') as f:
                 if float(f.read()) < oldest_accessed_dir["access_time"]:
                     timestamp = f.read()
                     if timestamp == '':
                         continue
-                    oldest_accessed_dir = {"dir": path, "access_time": float(timestamp)}
-    
+                    oldest_accessed_dir = {"dir": p, "access_time": float(timestamp)}
     return total_size, oldest_accessed_dir["dir"]
 
 
 def cleanup():
     with lock:
         print("Checking visualization cache...")
-        # Max tmp size is 50MB
-        max_size = 50000000
+        # Max tmp size is 500MB
+        max_size = 500000000
         folder_size, oldest_dir = scan_tmp_directory()
         while folder_size > max_size:
             print(f"Maximum cache size reached. Deleting {os.path.basename(oldest_dir)}.")
diff --git a/requirements.txt b/requirements.txt
index ba0bfb1..09b2fd3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-spacy==2.3.2
+spacy==2.*
 mmif-python==1.0.8
-mmif-python[utils]==1.0.8
+lapps
 flask-session
 flask[async]
 opencv-python==4.*

From aeb7f41a7b82e2e172cd12f3f3dc8b099e2f60e7 Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Sun, 27 Aug 2023 13:26:22 -0400
Subject: [PATCH 07/13] fixed all hard-coded paths

---
 app.py        | 48 ++++++++++++++------------
 cache.py      |  4 ++-
 iiif_utils.py |  4 ++-
 ocr.py        | 43 +++++++++++++-----------
 utils.py      | 93 ++++++++++++++++++++-------------------------------
 5 files changed, 93 insertions(+), 99 deletions(-)

diff --git a/app.py b/app.py
index 38bbc9d..48c2ad0 100644
--- a/app.py
+++ b/app.py
@@ -1,16 +1,17 @@
 import os
-import pathlib
-import sys
 import secrets
-import shortuuid
+import sys
+from pathlib import Path
 from threading import Thread
 
-from flask import request, render_template, flash, redirect, send_from_directory, session, redirect
-from werkzeug.utils import secure_filename
+import shortuuid
+from flask import request, render_template, flash, send_from_directory, session, redirect
 from mmif.serialize import Mmif
 
-from utils import app, render_ocr, get_media, prep_annotations, prepare_ocr_visualization
+import cache
 from cache import set_last_access, cleanup
+from utils import app, render_ocr, get_media, prep_annotations, prepare_ocr_visualization
+
 
 @app.route('/')
 def index():
@@ -21,7 +22,7 @@ def index():
 def ocr():
     try:
         data = dict(request.json)
-        mmif_str = open(os.path.join("/app/static/tmp", data["mmif_id"], "file.mmif")).read()
+        mmif_str = open(cache.get_cache_path() / data["mmif_id"] / "file.mmif").read()
         mmif = Mmif(mmif_str)
         ocr_view = mmif.get_view_by_id(data["view_id"])
         return prepare_ocr_visualization(mmif, ocr_view, data["mmif_id"])
@@ -33,10 +34,11 @@ def ocr():
 def ocrpage():
     data = request.json
     try:
-        return (render_ocr(data["mmif_id"], data['vid_path'], data["view_id"], data["page_number"]))
+        return render_ocr(data["mmif_id"], data['vid_path'], data["view_id"], data["page_number"])
     except Exception as e:
         return f'<p class="error">Unexpected error of type {type(e)}: {e}</h1>'
 
+
 @app.route('/upload', methods=['GET', 'POST'])
 def upload():
     # NOTE. Uses of flash() originally gaven a RuntimeError (The session is
@@ -58,13 +60,14 @@ def upload():
             return redirect(request.url)
         if file:
             return upload_file(file)
-        
+
     return render_template('upload.html')
 
-@app.route('/display/<id>')
-def display(id):
+
+@app.route('/display/<viz_id>')
+def display(viz_id):
     try:
-        path = os.path.join("/app/static/tmp", id)
+        path = cache.get_cache_path() / viz_id
         set_last_access(path)
         with open(os.path.join(path, "index.html")) as f:
             html_file = f.read()
@@ -73,10 +76,12 @@ def display(id):
         flash("File not found -- please upload again (it may have been deleted to clear up cache space).")
         return redirect("/upload")
 
+
 @app.route('/uv/<path:path>')
 def send_js(path):
     return send_from_directory("uv", path)
 
+
 def render_mmif(mmif_str):
     mmif = Mmif(mmif_str)
     media = get_media(mmif)
@@ -84,11 +89,13 @@ def render_mmif(mmif_str):
     return render_template('player.html',
                            media=media, annotations=annotations)
 
+
 def upload_file(file):
     # Save file locally
-    id = shortuuid.uuid()
-    session["mmif_id"] = id
-    path = os.path.join("/app/static/tmp", id)
+    uuid = shortuuid.uuid()
+    session["mmif_id"] = uuid
+    app.logger.debug(uuid)
+    path = Path(app.root_path) / 'static' / 'tmp' / uuid
     os.makedirs(path)
     set_last_access(path)
     file.save(os.path.join(path, "file.mmif"))
@@ -105,16 +112,15 @@ def upload_file(file):
 
     agent = request.headers.get('User-Agent')
     if 'curl' in agent.lower():
-        return f"Document ID is {id}\nYou can access the visualized file at /display/{id}\n"
-    return redirect(f"/display/{id}", code=302)
+        return f"Visualization ID is {uuid}\nYou can access the visualized file at /display/{uuid}\n"
+    return redirect(f"/display/{uuid}", code=301)
 
 
 if __name__ == '__main__':
     # Make path for temp files
-    tmp_path = pathlib.Path(__file__).parent /'static'/'tmp'
-    if not os.path.exists(tmp_path):
-        os.makedirs(tmp_path)
-
+    cache_path = cache.get_cache_path()
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
 
     # to avoid runtime errors for missing keys when using flash()
     alphabet = 'abcdefghijklmnopqrstuvwxyz1234567890'
diff --git a/cache.py b/cache.py
index bcb84cf..951604a 100644
--- a/cache.py
+++ b/cache.py
@@ -4,11 +4,13 @@
 import threading
 import pathlib
 
+from utils import app
+
 lock = threading.Lock()
 
 
 def get_cache_path():
-    return pathlib.Path(__file__).parent.parent / "static" / "tmp"
+    return pathlib.Path(app.root_path) / "static" / "tmp"
 
 
 def set_last_access(path):
diff --git a/iiif_utils.py b/iiif_utils.py
index 5ecad94..5e603e9 100644
--- a/iiif_utils.py
+++ b/iiif_utils.py
@@ -8,6 +8,7 @@
 from flask import url_for, session
 from mmif import AnnotationTypes, DocumentTypes, Mmif
 import datetime
+import cache
 
 
 def generate_iiif_manifest(in_mmif: mmif.Mmif):
@@ -107,7 +108,8 @@ def add_structure_from_timeframe(in_mmif: Mmif, iiif_json: Dict):
 
 def save_manifest(iiif_json: Dict) -> str:
     # generate a iiif manifest and save output file
-    manifest = tempfile.NamedTemporaryFile('w', dir=str(pathlib.Path(__file__).parent /'static/tmp'/session["mmif_id"]), suffix='.json', delete=False)
+    manifest = tempfile.NamedTemporaryFile(
+        'w', dir=str(cache.get_cache_path() / viz_id), suffix='.json', delete=False)
     json.dump(iiif_json, manifest, indent=4)
     return manifest.name
 
diff --git a/ocr.py b/ocr.py
index 202ea00..9b4ab08 100644
--- a/ocr.py
+++ b/ocr.py
@@ -10,6 +10,8 @@
 from flask import render_template
 from mmif.utils.video_document_helper import convert_timepoint, convert_timeframe
 
+import cache
+
 
 class OCRFrame():
     """Class representing an (aligned or otherwise) set of OCR annotations for a single frame"""
@@ -59,7 +61,6 @@ def add_bounding_box(self, anno, mmif):
         if anno.properties.get("boxType") and anno.properties.get("boxType") not in self.boxtypes:
             self.boxtypes.append(anno.properties.get("boxType"))
 
-
     def add_timeframe(self, anno, mmif):
         start, end = convert_timeframe(mmif, anno, "frames")
         start_secs, end_secs = convert_timeframe(mmif, anno, "seconds")
@@ -86,7 +87,7 @@ def get_ocr_frames(view, mmif, fps):
         for alignment in view.get_annotations(full_alignment_type[0]):
             source = find_annotation(alignment.properties["source"], view, mmif)
             target = find_annotation(alignment.properties["target"], view, mmif)
-            
+
             frame = OCRFrame(source, mmif)
             i = frame.frame_num if frame.frame_num is not None else frame.range
             if i in frames.keys():
@@ -124,11 +125,12 @@ def paginate(frames_list):
 
     return {i: page for (i, page) in enumerate(pages)}
 
+
 def render_ocr(mmif_id, vid_path, view_id, page_number):
     """Iterate through frames and display the contents/alignments."""
     # Path for storing temporary images generated by cv2
     cv2_vid = cv2.VideoCapture(vid_path)
-    f = open(os.path.join("/app/static/tmp", mmif_id, f"{view_id}-pages.json"))
+    f = open(cache.get_cache_path() / mmif_id / f"{view_id}-pages.json")
     frames_pages = json.load(f)
     page = frames_pages[str(page_number)]
     prev_frame_cap = None
@@ -143,10 +145,10 @@ def render_ocr(mmif_id, vid_path, view_id, page_number):
             raise FileNotFoundError(f"Video file {vid_path} not found!")
 
         # Double check histogram similarity of "repeat" frames -- if they're significantly different, un-mark as repeat
-        if prev_frame_cap is not None and frame["repeat"] and not is_duplicate_image(prev_frame_cap, frame_cap, cv2_vid):
+        if prev_frame_cap is not None and frame["repeat"] and not is_duplicate_image(prev_frame_cap, frame_cap,
+                                                                                     cv2_vid):
             frame["repeat"] = False
-        with tempfile.NamedTemporaryFile(
-                prefix=path, suffix=".jpg", delete=False) as tf:
+        with tempfile.NamedTemporaryFile(dir=str(path), suffix=".jpg", delete=False) as tf:
             cv2.imwrite(tf.name, frame_cap)
             # "id" is just the name of the temp image file
             frame["id"] = pathlib.Path(tf.name).name
@@ -158,12 +160,12 @@ def render_ocr(mmif_id, vid_path, view_id, page_number):
                            page=page,
                            n_pages=len(frames_pages),
                            page_number=str(page_number),
-                           mmif_id = mmif_id)
+                           mmif_id=mmif_id)
 
 
 def make_image_directory(mmif_id):
     # Make path for temp OCR image files or clear image files if it exists
-    path = os.path.join("/app/static/tmp", mmif_id, "img/")
+    path = cache.get_cache_path() / mmif_id / "img"
     if os.path.exists(path):
         shutil.rmtree(path)
     os.makedirs(path)
@@ -185,15 +187,15 @@ def find_duplicates(frames_list, cv2_vid):
 
 def is_duplicate_ocr_frame(prev_frame, frame):
     if not prev_frame:
-        return False 
+        return False
     if prev_frame.get("boxtypes") != frame.get("boxtypes"):
         return False
-    if abs(len(prev_frame.get("boxes"))-len(frame.get("boxes"))) > 3:
+    if abs(len(prev_frame.get("boxes")) - len(frame.get("boxes"))) > 3:
         return False
     # Check Boundingbox distances
     rounded_prev = round_boxes(prev_frame.get("boxes"))
     for box in round_boxes(frame.get("boxes")):
-        if box in rounded_prev and frame["secs"]-prev_frame["secs"] < 10:
+        if box in rounded_prev and frame["secs"] - prev_frame["secs"] < 10:
             return True
     # Check overlap in text
     prev_text, text = set(prev_frame.get("text")), set(frame.get("text"))
@@ -201,16 +203,16 @@ def is_duplicate_ocr_frame(prev_frame, frame):
         return True
     return False
 
-def is_duplicate_image(prev_frame, frame, cv2_vid):
 
+def is_duplicate_image(prev_frame, frame, cv2_vid):
     # Convert it to HSV
     img1_hsv = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2HSV)
     img2_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
 
     # Calculate the histogram and normalize it
-    hist_img1 = cv2.calcHist([img1_hsv], [0,1], None, [180,256], [0,180,0,256])
+    hist_img1 = cv2.calcHist([img1_hsv], [0, 1], None, [180, 256], [0, 180, 0, 256])
     cv2.normalize(hist_img1, hist_img1, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX);
-    hist_img2 = cv2.calcHist([img2_hsv], [0,1], None, [180,256], [0,180,0,256])
+    hist_img2 = cv2.calcHist([img2_hsv], [0, 1], None, [180, 256], [0, 180, 0, 256])
     cv2.normalize(hist_img2, hist_img2, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX);
 
     # Find the metric value
@@ -218,14 +220,13 @@ def is_duplicate_image(prev_frame, frame, cv2_vid):
     return metric_val < 50
 
 
-
 def round_boxes(boxes):
     # To account for jittery bounding boxes in OCR annotations
     rounded_boxes = []
     for box in boxes:
         rounded_box = []
         for coord in box[2]:
-            rounded_box.append(round(coord/100)*100)
+            rounded_box.append(round(coord / 100) * 100)
         rounded_boxes.append(rounded_box)
     return rounded_boxes
 
@@ -237,7 +238,8 @@ def get_ocr_views(mmif):
     for view in mmif.views:
         for anno_type, anno in view.metadata.contains.items():
             # Annotation belongs to a CV view if it is a TimeFrame/BB and it refers to a VideoDocument
-            if anno_type.shortname in required_types and mmif.get_document_by_id(anno["document"]).at_type.shortname == "VideoDocument":
+            if anno_type.shortname in required_types and mmif.get_document_by_id(
+                    anno["document"]).at_type.shortname == "VideoDocument":
                 views.append(view)
                 continue
             # TODO: Couldn't find a simple way to show if an alignment view is a CV/Frames-type view
@@ -246,7 +248,8 @@ def get_ocr_views(mmif):
                 continue
     return views
 
-def save_json(dict, view_id, mmif_id):
-    path = os.path.join("/app/static/tmp", mmif_id, f"{view_id}-pages.json")
+
+def save_json(data, view_id, mmif_id):
+    path = cache.get_cache_path() / mmif_id / f"{view_id}-pages.json"
     with open(path, 'w') as f:
-        json.dump(dict, f)
\ No newline at end of file
+        json.dump(data, f)
diff --git a/utils.py b/utils.py
index 2c3536c..7c49598 100644
--- a/utils.py
+++ b/utils.py
@@ -1,23 +1,16 @@
-import os
-import cv2
-from io import StringIO, BytesIO
-# from string import Template
 from collections import Counter
+from datetime import timedelta
+from io import StringIO
 
-import displacy
-import tempfile
-
-from flask import Flask, render_template, session
-from werkzeug.utils import secure_filename
-
-from mmif.serialize import Mmif, View
+from flask import Flask, session
+from lapps.discriminators import Uri
+from mmif import DocumentTypes
 from mmif.serialize.annotation import Text
 from mmif.vocabulary import AnnotationTypes
-from lapps.discriminators import Uri
+
+import displacy
 from iiif_utils import generate_iiif_manifest
 from ocr import *
-from datetime import timedelta
-
 
 # Get Properties from MMIF file ---
 
@@ -25,16 +18,17 @@
 app = Flask(__name__, static_folder='static', static_url_path='')
 app.secret_key = 'your_secret_key_here'
 
+
 def get_alignments(alignment_view):
-    vtt_file = tempfile.NamedTemporaryFile('w', dir=f"/app/static/tmp/{session['mmif_id']}", suffix='.vtt', delete=False)
+    vtt_file = tempfile.NamedTemporaryFile('w', dir=str(cache.get_cache_path() / session['mmif_id']), suffix='.vtt', delete=False)
     vtt_file.write("WEBVTT\n\n")
     annotations = alignment_view.annotations
-    timeframe_at_type = [at_type for at_type in alignment_view.metadata.contains if at_type.shortname == "TimeFrame" ][0]
+    timeframe_at_type = [at_type for at_type in alignment_view.metadata.contains if at_type.shortname == "TimeFrame"][0]
     timeunit = alignment_view.metadata.contains[timeframe_at_type]["timeUnit"]
     # TODO: wanted to use "mmif.get_alignments(AnnotationTypes.TimeFrame, Uri.TOKEN)"
     # but that gave errors so I gave up on it
-    token_idx = {a.id:a for a in annotations if a.at_type.shortname == "Token"}
-    timeframe_idx = {a.id:a for a in annotations if a.at_type.shortname == "TimeFrame"}
+    token_idx = {a.id: a for a in annotations if a.at_type.shortname == "Token"}
+    timeframe_idx = {a.id: a for a in annotations if a.at_type.shortname == "TimeFrame"}
     alignments = [a for a in annotations if a.at_type.shortname == "Alignment"]
     vtt_start = None
     texts = []
@@ -59,6 +53,7 @@ def get_alignments(alignment_view):
                 texts = []
     return vtt_file
 
+
 def build_alignment(alignment, token_idx, timeframe_idx):
     target = alignment.properties['target']
     source = alignment.properties['source']
@@ -70,6 +65,7 @@ def build_alignment(alignment, token_idx, timeframe_idx):
         text = token.properties['word']
         return start, end, text
 
+
 def get_media(mmif):
     # Returns a list of tuples, one for each element in the documents list of
     # the MMIF object, following the order in that list. Each tuple has four
@@ -77,21 +73,20 @@ def get_media(mmif):
     # visualization.
     media = []
     for document in mmif.documents:
-        doc_type = get_document_type_short_form(document)
-        doc_path = document.location
-        print('>>>', doc_path)
-        if doc_type == 'Text':
+        doc_path = document.location_path()
+        app.logger.debug(f"MMIF on AV asset: {doc_path}")
+        if document.at_type == DocumentTypes.TextDocument:
             html = html_text(doc_path)
-        elif doc_type == 'Video':
+        elif document.at_type == DocumentTypes.VideoDocument:
             fa_views = get_alignment_views(mmif)
             fa_view = fa_views[0] if fa_views else None
             html = html_video(doc_path, fa_view)
-        elif doc_type == 'Audio':
+        elif document.at_type == DocumentTypes.AudioDocument:
             html = html_audio(doc_path)
-        elif doc_type == 'Image':
+        elif document.at_type == DocumentTypes.ImageDocument:
             boxes = get_boxes(mmif)
             html = html_img(doc_path, boxes)
-        media.append((doc_type, document.id, doc_path, html))
+        media.append((document.at_type, document.id, doc_path, html))
     manifest_filename = generate_iiif_manifest(mmif)
     man = os.path.basename(manifest_filename)
     temp = render_template("uv_player.html", manifest=man, mmif_id=session["mmif_id"])
@@ -121,12 +116,6 @@ def get_boxes(mmif):
     return boxes
 
 
-def get_document_type_short_form(document):
-    """Returns 'Video', 'Text', 'Audio' or 'Image' from the document type of
-    the document."""
-    document_type = document.at_type.shortname
-    return document_type[:-8]
-
 def prep_annotations(mmif):
     """Prepare annotations from the views, and return a list of pairs of tabname
     and tab content. The first tab is alway the full MMIF pretty print."""
@@ -160,14 +149,6 @@ def prep_annotations(mmif):
     return tabs
 
 
-def get_video_path(mmif):
-    media = get_media(mmif)
-    for file in media:
-        if file[0] == "Video":
-            return file[2]
-    return None    
-
-
 def create_info(mmif):
     s = StringIO('Howdy')
     for document in mmif.documents:
@@ -197,7 +178,7 @@ def create_annotation_tables(mmif):
                 % (view.id, view.metadata.app, status, len(view.annotations)))
         s.write("<blockquote>\n")
         s.write("<table cellspacing=0 cellpadding=5 border=1>\n")
-        limit_len = lambda str : str[:500] + "  . . .  }" if len(str) > 500 else str
+        limit_len = lambda str: str[:500] + "  . . .  }" if len(str) > 500 else str
         for annotation in view.annotations:
             s.write('  <tr>\n')
             s.write('    <td>%s</td>\n' % annotation.id)
@@ -234,7 +215,6 @@ def get_alignment_views(mmif):
     return views
 
 
-
 # Remder Media as HTML ------------
 
 def html_video(vpath, vtt_srcview=None):
@@ -255,19 +235,10 @@ def html_video(vpath, vtt_srcview=None):
 
 
 def html_text(tpath):
-    """Return the conent of the text document, but with some HTML tags added."""
+    """Return the content of the text document, but with some HTML tags added."""
     if not os.path.isfile(tpath):
-        # This is to fix a problem when running this from a local machine where
-        # /data/text may not be available (it always is available from the
-        # container). The same problem occurs in displacy/__init__.py.
-        if tpath.startswith('file:///'):
-            tpath = tpath[8:]
-        else:
-            # this should not happen anymore, but keeping it anyway
-            tpath = tpath[1:]
-        tpath = os.path.join(app.root_path, 'static', tpath)
+        raise FileNotFoundError(f"File not found: {tpath}")
     with open(tpath) as t_file:
-        #return f"<pre width=\"100%\">\n{t_file.read()}\n</pre>"
         content = t_file.read().replace("\n", "<br/>\n")
         return f"{content}\n"
 
@@ -290,11 +261,13 @@ def url2posix(path):
         path = path[7:]
     return path
 
+
 # Interactive MMIF Tab -----------
 
 def render_interactive_mmif(mmif):
     return render_template('interactive.html', mmif=mmif, aligned_views=get_aligned_views(mmif))
 
+
 # Functions for checking if view can be rendered with alignment highlighting
 def get_aligned_views(mmif):
     """Return list of properly aligned views (for tree display)"""
@@ -305,6 +278,7 @@ def get_aligned_views(mmif):
                 aligned_views.append(view.id)
     return aligned_views
 
+
 def check_view_alignment(annotations):
     anno_stack = []
     for annotation in annotations:
@@ -313,22 +287,26 @@ def check_view_alignment(annotations):
         else:
             anno_stack.append(annotation.id)
         if len(anno_stack) == 3:
-            if type(anno_stack[0]) == str or not (anno_stack[0]["source"] in anno_stack and anno_stack[0]["target"] in anno_stack):
+            if type(anno_stack[0]) == str or not (
+                    anno_stack[0]["source"] in anno_stack and anno_stack[0]["target"] in anno_stack):
                 return False
             anno_stack = []
     return True
 
+
 # NER Tools ----------------------
 
 def get_ner_views(mmif):
     return [v for v in mmif.views if Uri.NE in v.metadata.contains]
 
+
 def view_to_vtt(alignment_view):
     """Write alignments to a file in VTT style and return the filename."""
     vtt_file = get_alignments(alignment_view)
     return vtt_file.name
     # return os.sep.join(vtt_file.name.split(os.sep)[-3:])
 
+
 def create_ner_visualization(mmif, view):
     metadata = view.metadata.contains.get(Uri.NE)
     try:
@@ -338,6 +316,8 @@ def create_ner_visualization(mmif, view):
     except KeyError as e:
         # the view's entities refer to more than one text document (tessearct)
         pass
+
+
 def get_status(view):
     return 'ERROR' if 'message' in view.metadata.error else 'OKAY'
 
@@ -353,12 +333,13 @@ def get_properties(annotation):
         props_list.append("%s=%s" % (prop, val))
     return '{ %s }' % ', '.join(props_list)
 
+
 # OCR Tools ----------------------
 
 def prepare_ocr_visualization(mmif, view, mmif_id):
     """ Visualize OCR by extracting image frames with BoundingBoxes from video"""
     # frames, text_docs, alignments = {}, {}, {}
-    vid_path = get_video_path(mmif)
+    vid_path = mmif.get_documents_by_type(DocumentTypes.VideoDocument)[0].location_path()
     cv2_vid = cv2.VideoCapture(vid_path)
     fps = cv2_vid.get(cv2.CAP_PROP_FPS)
 
@@ -370,4 +351,4 @@ def prepare_ocr_visualization(mmif, view, mmif_id):
     frames_pages = paginate(frames_list)
     # Save page list as temp file
     save_json(frames_pages, view.id, mmif_id)
-    return render_ocr(mmif_id, vid_path, view.id, 0)
\ No newline at end of file
+    return render_ocr(mmif_id, vid_path, view.id, 0)

From c86e1e2f09736b86d1a78a7dd24d720b7cb4c3bf Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Sun, 27 Aug 2023 15:50:17 -0400
Subject: [PATCH 08/13] removed reliance on browser session, replaced uuid with
 sha1 hash (#26)

---
 app.py        | 38 ++++++++++++++++++--------------------
 cache.py      |  6 +++++-
 iiif_utils.py | 12 ++++++------
 utils.py      | 47 ++++++++++++++++++++---------------------------
 4 files changed, 49 insertions(+), 54 deletions(-)

diff --git a/app.py b/app.py
index 48c2ad0..5e47e18 100644
--- a/app.py
+++ b/app.py
@@ -1,16 +1,15 @@
+import hashlib
 import os
 import secrets
 import sys
-from pathlib import Path
 from threading import Thread
 
-import shortuuid
-from flask import request, render_template, flash, send_from_directory, session, redirect
+from flask import request, render_template, flash, send_from_directory, redirect
 from mmif.serialize import Mmif
 
 import cache
 from cache import set_last_access, cleanup
-from utils import app, render_ocr, get_media, prep_annotations, prepare_ocr_visualization
+from utils import app, render_ocr, documents_to_htmls, prep_annotations, prepare_ocr_visualization
 
 
 @app.route('/')
@@ -82,29 +81,28 @@ def send_js(path):
     return send_from_directory("uv", path)
 
 
-def render_mmif(mmif_str):
+def render_mmif(mmif_str, viz_id):
     mmif = Mmif(mmif_str)
-    media = get_media(mmif)
-    annotations = prep_annotations(mmif)
+    media = documents_to_htmls(mmif, viz_id)
+    annotations = prep_annotations(mmif, viz_id)
     return render_template('player.html',
                            media=media, annotations=annotations)
 
 
-def upload_file(file):
+def upload_file(in_mmif):
     # Save file locally
-    uuid = shortuuid.uuid()
-    session["mmif_id"] = uuid
-    app.logger.debug(uuid)
-    path = Path(app.root_path) / 'static' / 'tmp' / uuid
-    os.makedirs(path)
+    in_mmif_bytes = in_mmif.read()
+    in_mmif_str = in_mmif_bytes.decode('utf-8')
+    viz_id = hashlib.sha1(in_mmif_bytes).hexdigest()
+    app.logger.debug(viz_id)
+    path = cache.get_cache_path() / viz_id
+    os.makedirs(path, exist_ok=True)
     set_last_access(path)
-    file.save(os.path.join(path, "file.mmif"))
-    with open(os.path.join(path, "file.mmif")) as fh:
-        mmif_str = fh.read()
-    html_page = render_mmif(mmif_str)
+    with open(path / 'file.mmif', 'w') as in_mmif_file:
+        in_mmif_file.write(in_mmif_str)
+    html_page = render_mmif(in_mmif_str, viz_id)
     with open(os.path.join(path, "index.html"), "w") as f:
         f.write(html_page)
-
     # Perform cleanup
     t = Thread(target=cleanup)
     t.daemon = True
@@ -112,8 +110,8 @@ def upload_file(file):
 
     agent = request.headers.get('User-Agent')
     if 'curl' in agent.lower():
-        return f"Visualization ID is {uuid}\nYou can access the visualized file at /display/{uuid}\n"
-    return redirect(f"/display/{uuid}", code=301)
+        return f"Visualization ID is {viz_id}\nYou can access the visualized file at /display/{viz_id}\n"
+    return redirect(f"/display/{viz_id}", code=301)
 
 
 if __name__ == '__main__':
diff --git a/cache.py b/cache.py
index 951604a..4892893 100644
--- a/cache.py
+++ b/cache.py
@@ -10,7 +10,11 @@
 
 
 def get_cache_path():
-    return pathlib.Path(app.root_path) / "static" / "tmp"
+    return pathlib.Path(app.static_folder) / "tmp"
+
+
+def get_cache_relpath(full_path):
+    return str(full_path)[len(app.static_folder):]
 
 
 def set_last_access(path):
diff --git a/iiif_utils.py b/iiif_utils.py
index 5e603e9..7be9eb8 100644
--- a/iiif_utils.py
+++ b/iiif_utils.py
@@ -1,17 +1,17 @@
+import datetime
 import json
 import os
-import pathlib
 import tempfile
 from typing import Dict
 
 import mmif
-from flask import url_for, session
+from flask import url_for
 from mmif import AnnotationTypes, DocumentTypes, Mmif
-import datetime
+
 import cache
 
 
-def generate_iiif_manifest(in_mmif: mmif.Mmif):
+def generate_iiif_manifest(in_mmif: mmif.Mmif, viz_id):
     iiif_json = {
         "@context": "http://iiif.io/api/presentation/2/context.json",
         "id": "http://0.0.0.0:5000/mmif_example_manifest.json",
@@ -29,7 +29,7 @@ def generate_iiif_manifest(in_mmif: mmif.Mmif):
     }
     add_canvas_from_documents(in_mmif, iiif_json)
     add_structure_from_timeframe(in_mmif, iiif_json)
-    return save_manifest(iiif_json)
+    return save_manifest(iiif_json, viz_id)
 
 
 def add_canvas_from_documents(in_mmif, iiif_json):
@@ -106,7 +106,7 @@ def add_structure_from_timeframe(in_mmif: Mmif, iiif_json: Dict):
         iiif_json["structures"].append(view_range)
 
 
-def save_manifest(iiif_json: Dict) -> str:
+def save_manifest(iiif_json: Dict, viz_id) -> str:
     # generate a iiif manifest and save output file
     manifest = tempfile.NamedTemporaryFile(
         'w', dir=str(cache.get_cache_path() / viz_id), suffix='.json', delete=False)
diff --git a/utils.py b/utils.py
index 7c49598..ff18ddc 100644
--- a/utils.py
+++ b/utils.py
@@ -2,7 +2,7 @@
 from datetime import timedelta
 from io import StringIO
 
-from flask import Flask, session
+from flask import Flask
 from lapps.discriminators import Uri
 from mmif import DocumentTypes
 from mmif.serialize.annotation import Text
@@ -19,8 +19,11 @@
 app.secret_key = 'your_secret_key_here'
 
 
-def get_alignments(alignment_view):
-    vtt_file = tempfile.NamedTemporaryFile('w', dir=str(cache.get_cache_path() / session['mmif_id']), suffix='.vtt', delete=False)
+def asr_alignments_to_vtt(alignment_view, viz_id):
+    vtt_filename = cache.get_cache_path() / viz_id / f"{alignment_view.id.replace(':', '-')}.vtt" 
+    if vtt_filename.exists():
+        return str(vtt_filename)
+    vtt_file = open(vtt_filename, 'w')
     vtt_file.write("WEBVTT\n\n")
     annotations = alignment_view.annotations
     timeframe_at_type = [at_type for at_type in alignment_view.metadata.contains if at_type.shortname == "TimeFrame"][0]
@@ -51,7 +54,7 @@ def get_alignments(alignment_view):
                 vtt_file.write(f'{vtt_start} --> {vtt_end}\n{" ".join(texts)}\n\n')
                 vtt_start = None
                 texts = []
-    return vtt_file
+    return vtt_file.name
 
 
 def build_alignment(alignment, token_idx, timeframe_idx):
@@ -66,7 +69,7 @@ def build_alignment(alignment, token_idx, timeframe_idx):
         return start, end, text
 
 
-def get_media(mmif):
+def documents_to_htmls(mmif, viz_id):
     # Returns a list of tuples, one for each element in the documents list of
     # the MMIF object, following the order in that list. Each tuple has four
     # elements: document type, document identifier, document path and the HTML
@@ -80,16 +83,16 @@ def get_media(mmif):
         elif document.at_type == DocumentTypes.VideoDocument:
             fa_views = get_alignment_views(mmif)
             fa_view = fa_views[0] if fa_views else None
-            html = html_video(doc_path, fa_view)
+            html = html_video(viz_id, doc_path, fa_view)
         elif document.at_type == DocumentTypes.AudioDocument:
             html = html_audio(doc_path)
         elif document.at_type == DocumentTypes.ImageDocument:
             boxes = get_boxes(mmif)
             html = html_img(doc_path, boxes)
         media.append((document.at_type, document.id, doc_path, html))
-    manifest_filename = generate_iiif_manifest(mmif)
+    manifest_filename = generate_iiif_manifest(mmif, viz_id)
     man = os.path.basename(manifest_filename)
-    temp = render_template("uv_player.html", manifest=man, mmif_id=session["mmif_id"])
+    temp = render_template("uv_player.html", manifest=man, mmif_id=viz_id)
     media.append(('UV', "", "", temp))
     return media
 
@@ -116,7 +119,7 @@ def get_boxes(mmif):
     return boxes
 
 
-def prep_annotations(mmif):
+def prep_annotations(mmif, viz_id):
     """Prepare annotations from the views, and return a list of pairs of tabname
     and tab content. The first tab is alway the full MMIF pretty print."""
     tabs = [("Info", "<pre>" + create_info(mmif) + "</pre>"),
@@ -127,7 +130,7 @@ def prep_annotations(mmif):
     # stuff; it does a loop but for now we assume there is just one file with
     # alignments (generated by Kaldi)
     for fa_view in get_alignment_views(mmif):
-        vtt_file = view_to_vtt(fa_view)
+        vtt_file = asr_alignments_to_vtt(fa_view, viz_id)
         tabs.append(("WebVTT", '<pre>' + open(vtt_file).read() + '</pre>'))
     ner_views = get_ner_views(mmif)
     use_id = True if len(ner_views) > 1 else False
@@ -144,7 +147,7 @@ def prep_annotations(mmif):
         if not ocr_view.annotations:
             continue
         tabname = "Frames-%s" % ocr_view.id
-        visualization = render_template("pre-ocr.html", view_id=ocr_view.id, tabname=tabname, mmif_id=session["mmif_id"])
+        visualization = render_template("pre-ocr.html", view_id=ocr_view.id, tabname=tabname, mmif_id=viz_id)
         tabs.append((tabname, visualization))
     return tabs
 
@@ -217,19 +220,16 @@ def get_alignment_views(mmif):
 
 # Remder Media as HTML ------------
 
-def html_video(vpath, vtt_srcview=None):
+def html_video(viz_id, vpath, vtt_srcview=None):
     vpath = url2posix(vpath)
     html = StringIO()
-    html.write('<video id="vid" controls>\n')
+    html.write('<video id="vid" controls crossorigin="anonymous" >\n')
     html.write(f'    <source src=\"{vpath}\">\n')
     if vtt_srcview is not None:
-        vtt_path = view_to_vtt(vtt_srcview)
-        src = "/" + os.sep.join(vtt_path.split(os.sep)[-2:])
-        # use only basename because "static" directory is mapped to '' route by
-        # `static_url_path` param
-        # src = os.path.basename(vtt_path)
-
-        html.write(f'    <track kind="subtitles" srclang="en" src="{src}" label="English" default>\n')
+        vtt_path = asr_alignments_to_vtt(vtt_srcview, viz_id)
+        src = cache.get_cache_relpath(vtt_path)
+        app.logger.debug(f"VTT path: {vtt_path}")
+        html.write(f'    <track kind="captions" srclang="en" src="{src}" label="transcript" default/>\n')
     html.write("</video>\n")
     return html.getvalue()
 
@@ -300,13 +300,6 @@ def get_ner_views(mmif):
     return [v for v in mmif.views if Uri.NE in v.metadata.contains]
 
 
-def view_to_vtt(alignment_view):
-    """Write alignments to a file in VTT style and return the filename."""
-    vtt_file = get_alignments(alignment_view)
-    return vtt_file.name
-    # return os.sep.join(vtt_file.name.split(os.sep)[-3:])
-
-
 def create_ner_visualization(mmif, view):
     metadata = view.metadata.contains.get(Uri.NE)
     try:

From 31cbbe1a04b68f98375dc9f2fbb32efd27740b3f Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Sun, 27 Aug 2023 18:56:58 -0400
Subject: [PATCH 09/13] use cached index.html if exists, changed tab name for
 thumbnails

---
 app.py             | 30 ++++++++++++++++++------------
 ocr.py             | 17 +++++++----------
 templates/ocr.html |  4 ++--
 utils.py           |  2 +-
 4 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/app.py b/app.py
index 5e47e18..ecf7518 100644
--- a/app.py
+++ b/app.py
@@ -85,6 +85,7 @@ def render_mmif(mmif_str, viz_id):
     mmif = Mmif(mmif_str)
     media = documents_to_htmls(mmif, viz_id)
     annotations = prep_annotations(mmif, viz_id)
+    app.logger.debug(f"Prepared Annotations: {annotations.keys()}")
     return render_template('player.html',
                            media=media, annotations=annotations)
 
@@ -94,19 +95,24 @@ def upload_file(in_mmif):
     in_mmif_bytes = in_mmif.read()
     in_mmif_str = in_mmif_bytes.decode('utf-8')
     viz_id = hashlib.sha1(in_mmif_bytes).hexdigest()
-    app.logger.debug(viz_id)
+    app.logger.debug(f"Visualization ID: {viz_id}")
     path = cache.get_cache_path() / viz_id
-    os.makedirs(path, exist_ok=True)
-    set_last_access(path)
-    with open(path / 'file.mmif', 'w') as in_mmif_file:
-        in_mmif_file.write(in_mmif_str)
-    html_page = render_mmif(in_mmif_str, viz_id)
-    with open(os.path.join(path, "index.html"), "w") as f:
-        f.write(html_page)
-    # Perform cleanup
-    t = Thread(target=cleanup)
-    t.daemon = True
-    t.run()
+    app.logger.debug(f"Visualization Directory: {path}")
+    try:
+        os.makedirs(path)
+        set_last_access(path)
+        with open(path / 'file.mmif', 'w') as in_mmif_file:
+            in_mmif_file.write(in_mmif_str)
+        html_page = render_mmif(in_mmif_str, viz_id)
+        with open(os.path.join(path, "index.html"), "w") as f:
+            f.write(html_page)
+    except FileExistsError:
+        app.logger.debug("Visualization already cached")
+    finally:
+        # Perform cleanup
+        t = Thread(target=cleanup)
+        t.daemon = True
+        t.run()
 
     agent = request.headers.get('User-Agent')
     if 'curl' in agent.lower():
diff --git a/ocr.py b/ocr.py
index 9b4ab08..41aa30e 100644
--- a/ocr.py
+++ b/ocr.py
@@ -130,9 +130,9 @@ def render_ocr(mmif_id, vid_path, view_id, page_number):
     """Iterate through frames and display the contents/alignments."""
     # Path for storing temporary images generated by cv2
     cv2_vid = cv2.VideoCapture(vid_path)
-    f = open(cache.get_cache_path() / mmif_id / f"{view_id}-pages.json")
-    frames_pages = json.load(f)
-    page = frames_pages[str(page_number)]
+    tn_data_fname = cache.get_cache_path() / mmif_id / f"{view_id}-pages.json"
+    thumbnail_pages = json.load(open(tn_data_fname))
+    page = thumbnail_pages[str(page_number)]
     prev_frame_cap = None
     path = make_image_directory(mmif_id)
     for frame_num, frame in page:
@@ -154,13 +154,10 @@ def render_ocr(mmif_id, vid_path, view_id, page_number):
             frame["id"] = pathlib.Path(tf.name).name
         prev_frame_cap = frame_cap
 
-    return render_template('ocr.html',
-                           vid_path=vid_path,
-                           view_id=view_id,
-                           page=page,
-                           n_pages=len(frames_pages),
-                           page_number=str(page_number),
-                           mmif_id=mmif_id)
+    tn_page_html = render_template(
+        'ocr.html', vid_path=vid_path, view_id=view_id, page=page,
+        n_pages=len(thumbnail_pages), page_number=str(page_number), mmif_id=mmif_id)
+    return tn_page_html
 
 
 def make_image_directory(mmif_id):
diff --git a/templates/ocr.html b/templates/ocr.html
index 08916c8..ff584eb 100644
--- a/templates/ocr.html
+++ b/templates/ocr.html
@@ -130,10 +130,10 @@ <h4>
             "page_number": parseInt("{{page_number}}"),
             "mmif_id": "{{mmif_id}}"
         }
-        if (page == BACKWARD) {
+        if (page === BACKWARD) {
             data["page_number"] -= 1
         }
-        else if (page == FORWARD) {
+        else if (page === FORWARD) {
             data["page_number"] += 1
         }
         else {
diff --git a/utils.py b/utils.py
index ff18ddc..9971c96 100644
--- a/utils.py
+++ b/utils.py
@@ -146,7 +146,7 @@ def prep_annotations(mmif, viz_id):
     for ocr_view in ocr_views:
         if not ocr_view.annotations:
             continue
-        tabname = "Frames-%s" % ocr_view.id
+        tabname = "Thumbnails-%s" % ocr_view.id
         visualization = render_template("pre-ocr.html", view_id=ocr_view.id, tabname=tabname, mmif_id=viz_id)
         tabs.append((tabname, visualization))
     return tabs

From 4f109268c87bbc0ea08643bde99955263e94389c Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Sun, 27 Aug 2023 19:29:56 -0400
Subject: [PATCH 10/13] now can invalidate cache, added invalidation btn for
 single viz screen

---
 app.py                | 18 +++++++++++++++---
 cache.py              | 11 +++++++++++
 templates/player.html |  3 +++
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/app.py b/app.py
index ecf7518..3f11769 100644
--- a/app.py
+++ b/app.py
@@ -63,6 +63,18 @@ def upload():
     return render_template('upload.html')
 
 
+@app.route('/decache', methods=['GET', 'POST'])
+def invalidate_cache():
+    app.logger.debug(f"Request to invalidate cache on {request.args}")
+    if not request.args.get('viz_id'):
+        cache.invalidate_cache()
+        return redirect("/upload")
+    viz_id = request.args.get('viz_id')
+    in_mmif = open(cache.get_cache_path() / viz_id / 'file.mmif', 'rb').read()
+    cache.invalidate_cache([viz_id])
+    return upload_file(in_mmif)
+
+
 @app.route('/display/<viz_id>')
 def display(viz_id):
     try:
@@ -85,14 +97,14 @@ def render_mmif(mmif_str, viz_id):
     mmif = Mmif(mmif_str)
     media = documents_to_htmls(mmif, viz_id)
     annotations = prep_annotations(mmif, viz_id)
-    app.logger.debug(f"Prepared Annotations: {annotations.keys()}")
+    app.logger.debug(f"Prepared Annotations: {[annotation[0] for annotation in annotations]}")
     return render_template('player.html',
-                           media=media, annotations=annotations)
+                           media=media, viz_id=viz_id, annotations=annotations)
 
 
 def upload_file(in_mmif):
     # Save file locally
-    in_mmif_bytes = in_mmif.read()
+    in_mmif_bytes = in_mmif if isinstance(in_mmif, bytes) else in_mmif.read()
     in_mmif_str = in_mmif_bytes.decode('utf-8')
     viz_id = hashlib.sha1(in_mmif_bytes).hexdigest()
     app.logger.debug(f"Visualization ID: {viz_id}")
diff --git a/cache.py b/cache.py
index 4892893..7c9660b 100644
--- a/cache.py
+++ b/cache.py
@@ -17,6 +17,17 @@ def get_cache_relpath(full_path):
     return str(full_path)[len(app.static_folder):]
 
 
+def invalidate_cache(viz_ids):
+    if not viz_ids:
+        app.logger.debug("Invalidating entire cache.")
+        shutil.rmtree(get_cache_path())
+        os.makedirs(get_cache_path())
+    else:
+        for v in viz_ids:
+            app.logger.debug(f"Invalidating {v} from cache.")
+            shutil.rmtree(get_cache_path() / v)
+
+
 def set_last_access(path):
     with open(os.path.join(path, "last_access.txt"), "w") as f:
         f.write(str(time.time()))
diff --git a/templates/player.html b/templates/player.html
index 930b085..5da61c0 100644
--- a/templates/player.html
+++ b/templates/player.html
@@ -100,6 +100,9 @@
       <form action="/upload">
         <input type="submit" value="Upload another file" />
       </form>
+      <form action="/decache?viz_id={{ viz_id }}" method="POST">
+        <input type="submit" value="Invalidate cache and regenerate visualization" />  
+      </form>
     </div>
     <h1 class="title">Visualizing MMIF</h1>
     <div class="right"></div>

From bb2ff085b403e33e6f6c7f4543476a78d06de746 Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Sun, 27 Aug 2023 19:38:34 -0400
Subject: [PATCH 11/13] disabled RAW MMIF tab to speed up

---
 app.py   |  2 ++
 utils.py | 16 ++++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/app.py b/app.py
index 3f11769..5e53c65 100644
--- a/app.py
+++ b/app.py
@@ -96,6 +96,7 @@ def send_js(path):
 def render_mmif(mmif_str, viz_id):
     mmif = Mmif(mmif_str)
     media = documents_to_htmls(mmif, viz_id)
+    app.logger.debug(f"Prepared Media: {[m[0] for m in media]}")
     annotations = prep_annotations(mmif, viz_id)
     app.logger.debug(f"Prepared Annotations: {[annotation[0] for annotation in annotations]}")
     return render_template('player.html',
@@ -114,6 +115,7 @@ def upload_file(in_mmif):
         os.makedirs(path)
         set_last_access(path)
         with open(path / 'file.mmif', 'w') as in_mmif_file:
+            app.logger.debug(f"Writing original MMIF to {path / 'file.mmif'}")
             in_mmif_file.write(in_mmif_str)
         html_page = render_mmif(in_mmif_str, viz_id)
         with open(os.path.join(path, "index.html"), "w") as f:
diff --git a/utils.py b/utils.py
index 9971c96..d0eaa21 100644
--- a/utils.py
+++ b/utils.py
@@ -122,16 +122,22 @@ def get_boxes(mmif):
 def prep_annotations(mmif, viz_id):
     """Prepare annotations from the views, and return a list of pairs of tabname
     and tab content. The first tab is alway the full MMIF pretty print."""
-    tabs = [("Info", "<pre>" + create_info(mmif) + "</pre>"),
-            ("MMIF", "<pre>" + mmif.serialize(pretty=True) + "</pre>"),
-            ("Annotations", create_annotation_tables(mmif)),
-            ("Tree", render_interactive_mmif(mmif))]
+    tabs = []
+    tabs.append(("Info", "<pre>" + create_info(mmif) + "</pre>"))
+    app.logger.debug(f"Prepared INFO Tab: {tabs[-1][0]}")
+    # tabs.append(("MMIF", "<pre>" + mmif.serialize(pretty=True) + "</pre>"))
+    # app.logger.debug(f"Prepared RAW Tab: {tabs[-1][0]}")
+    tabs.append(("Annotations", create_annotation_tables(mmif)))
+    app.logger.debug(f"Prepared SUMMARY Tab: {tabs[-1][0]}")
+    tabs.append(("Tree", render_interactive_mmif(mmif)))
+    app.logger.debug(f"Prepared JSTREE Tab: {tabs[-1][0]}")
     # TODO: since this uses the same tab-name this will only show the same
     # stuff; it does a loop but for now we assume there is just one file with
     # alignments (generated by Kaldi)
     for fa_view in get_alignment_views(mmif):
         vtt_file = asr_alignments_to_vtt(fa_view, viz_id)
         tabs.append(("WebVTT", '<pre>' + open(vtt_file).read() + '</pre>'))
+        app.logger.debug(f"Prepared a VTT Tab: {tabs[-1][0]}")
     ner_views = get_ner_views(mmif)
     use_id = True if len(ner_views) > 1 else False
     for ner_view in ner_views:
@@ -140,6 +146,7 @@ def prep_annotations(mmif, viz_id):
         visualization = create_ner_visualization(mmif, ner_view)
         tabname = "Entities-%s" % ner_view.id if use_id else "Entities"
         tabs.append((tabname, visualization))
+        app.logger.debug(f"Prepared a displaCy Tab: {tabs[-1][0]}")
     # TODO: somewhat hackish
     ocr_views = get_ocr_views(mmif)
     use_id = True if len(ocr_views) > 1 else False
@@ -149,6 +156,7 @@ def prep_annotations(mmif, viz_id):
         tabname = "Thumbnails-%s" % ocr_view.id
         visualization = render_template("pre-ocr.html", view_id=ocr_view.id, tabname=tabname, mmif_id=viz_id)
         tabs.append((tabname, visualization))
+        app.logger.debug(f"Prepared a Thumbnails Tab: {tabs[-1][0]}")
     return tabs
 
 

From 4edb59c8590d2419d4c6ccf2339dc4a5cf527aaf Mon Sep 17 00:00:00 2001
From: Hayden McCormick <haydenwmccormick@gmail.com>
Date: Sat, 9 Sep 2023 13:57:18 -0400
Subject: [PATCH 12/13] Fix VTT formatting issues (#29)

---
 utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/utils.py b/utils.py
index d0eaa21..c248fd9 100644
--- a/utils.py
+++ b/utils.py
@@ -46,11 +46,13 @@ def asr_alignments_to_vtt(alignment_view, viz_id):
             start, end, text = start_end_text
             start_kwarg, end_kwarg = {timeunit: float(start)}, {timeunit: float(end)}
             start, end = timedelta(**start_kwarg), timedelta(**end_kwarg)
+            s_mins, s_secs = divmod(start.seconds, 60)
+            e_mins, e_secs = divmod(end.seconds, 60)
             if not vtt_start:
-                vtt_start = f'{start.seconds // 3600 :02d}:{start.seconds:02d}.{start.microseconds // 1000 :03d}'
+                vtt_start = f'{s_mins:02d}:{s_secs:02d}.{((s_secs - int(s_secs)) * 1000):03d}'
             texts.append(text)
             if len(texts) > 8:
-                vtt_end = f'{end.seconds // 3600 :02d}:{end.seconds:02d}.{end.microseconds // 1000 :03d}'
+                vtt_end = f'{e_mins:02d}:{e_secs:02d}.{((e_secs - int(e_secs)) * 1000):03d}'
                 vtt_file.write(f'{vtt_start} --> {vtt_end}\n{" ".join(texts)}\n\n')
                 vtt_start = None
                 texts = []

From 9a28864882467c6058edbc61a8c7b80fc72d551d Mon Sep 17 00:00:00 2001
From: Keigh Rim <keigh.rim@gmail.com>
Date: Tue, 12 Sep 2023 01:46:49 -0400
Subject: [PATCH 13/13] fixed document tab title format

---
 utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils.py b/utils.py
index c248fd9..b8718d9 100644
--- a/utils.py
+++ b/utils.py
@@ -91,7 +91,7 @@ def documents_to_htmls(mmif, viz_id):
         elif document.at_type == DocumentTypes.ImageDocument:
             boxes = get_boxes(mmif)
             html = html_img(doc_path, boxes)
-        media.append((document.at_type, document.id, doc_path, html))
+        media.append((document.at_type.shortname, document.id, doc_path, html))
     manifest_filename = generate_iiif_manifest(mmif, viz_id)
     man = os.path.basename(manifest_filename)
     temp = render_template("uv_player.html", manifest=man, mmif_id=viz_id)