Merge pull request #25 from clamsproject/14-clamsapp

Making this a true Clams App

.dockerignore

@@ -0,0 +1,17 @@
+.git/
+.github/
+.gitignore
+.dockerignore
+Dockerfile
+Containerfile
+bin/
+build/
+*~
+**/*.pyc
+**/__pyche__
+modeling/results*/
+modeling/vectorized*/
+modeling/annotations-gbh/
+modeling/frames/
+modeling/features/
+modeling/data/cpb-aacip-690722078b2.mp4

.gitignore

@@ -23,7 +23,8 @@
 .LSOverride
 
 # Icon must end with two \r
-Icon
+Icon
+
 
 # Thumbnails
 ._*
@@ -173,6 +174,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
+venv
 env/
 venv/
 ENV/
@@ -246,4 +248,21 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
-# End of https://www.toptal.com/developers/gitignore/api/python,macos,windows,linux
+# End of https://www.toptal.com/developers/gitignore/api/python,mac,windows,linux
+
+.idea
+
+# MacOS FInder information
+.DS_Store
+
+# Files created by data_ingestion
+modeling/features
+modeling/results*/*kfold*pt
+modeling/results*/*kfold*csv
+modeling/results*/*kfold*txt
+modeling/vectorized/*
+
+# Files used or created by classify.py
+modeling/data/*
+modeling/frames/*
+modeling/predictions.json

Containerfile

@@ -0,0 +1,43 @@
+FROM ghcr.io/clamsproject/clams-python-opencv4:1.0.9
+
+# See https://github.com/orgs/clamsproject/packages?tab=packages&q=clams-python for more base images
+# IF you want to automatically publish this image to the clamsproject organization, 
+# 1. you should have generated this template without --no-github-actions flag
+# 1. to add arm64 support, change relevant line in .github/workflows/container.yml 
+#     * NOTE that a lots of software doesn't install/compile or run on arm64 architecture out of the box 
+#     * make sure you locally test the compatibility of all software dependencies before using arm64 support 
+# 1. use a git tag to trigger the github action. You need to use git tag to properly set app version anyway
+
+################################################################################
+# DO NOT EDIT THIS SECTION
+ARG CLAMS_APP_VERSION
+ENV CLAMS_APP_VERSION ${CLAMS_APP_VERSION}
+################################################################################
+
+################################################################################
+# clams-python base images are based on debian distro
+# install more system packages as needed using the apt manager
+################################################################################
+
+RUN apt-get update && apt-get install -y wget
+
+################################################################################
+# main app installation
+
+RUN pip install --no-cache-dir torch==2.1.0
+RUN pip install --no-cache-dir torchvision==0.16.0
+
+# Getting the model at build time so we don't need to get it each time we start
+# a container. This is also because without it I ran into "Connection reset by peer"
+# errors once in a while.
+RUN wget https://download.pytorch.org/models/vgg16-397923af.pth
+RUN mkdir /root/.cache/torch /root/.cache/torch/hub /root/.cache/torch/hub/checkpoints
+RUN mv vgg16-397923af.pth /root/.cache/torch/hub/checkpoints
+
+WORKDIR /app
+
+COPY . /app
+
+# default command to run the CLAMS app in a production server 
+CMD ["python3", "app.py", "--production"]
+################################################################################

README.md

@@ -1,2 +1,125 @@
-# app-SWT-detection
-CLAMS app for detecting scenes with text from video input
+# Scene-with-text 
+
+
+## Description
+
+Proof of concept prototype for an app that extracts scenes with textual content. At the moment, it extracts slates, chyrons and credits.
+
+
+## User instructions
+
+General user instructions for CLAMS apps is available at [CLAMS Apps documentation](https://apps.clams.ai/clamsapp).
+
+
+### System requirements
+
+The preferred platform is Debian 10.13 or higher, but the code is known to run on MacOSX. GPU is not required but performance will be better with it. The main system packages needed are FFmpeg ([https://ffmpeg.org/](https://ffmpeg.org/)), OpenCV4 ([https://opencv.org/](https://opencv.org/)), and Python 3.8 or higher. 
+
+The easiest way to get these is to get the Docker [clams-python-opencv4](https://github.com/clamsproject/clams-python/pkgs/container/clams-python-opencv4) base image. For more details take a peek at the following container specifications:
+
+- [https://github.com/clamsproject/clams-python/blob/main/container/Containerfile](https://github.com/clamsproject/clams-python/blob/main/container/Containerfile)
+- [https://github.com/clamsproject/clams-python/blob/main/container/ffmpeg.containerfile](https://github.com/clamsproject/clams-python/blob/main/container/ffmpeg.containerfile)
+- [https://github.com/clamsproject/clams-python/blob/main/container/opencv4.containerfile](https://github.com/clamsproject/clams-python/blob/main/container/opencv4.containerfile)
+
+The following Python packages are needed: clams-python, ffmpeg-python, opencv-python-rolling, torch and torchvision:
+
+```bash
+pip install clams-python==1.0.9 ffmpeg-python==0.2.* opencv-python-rolling
+pip install torch==2.1.0 torchvision==0.16.0
+```
+
+The installs in the first line are part of the clams-python-opencv4 image, the torch and torchvision packages need to be installed in addition (see the `Containerfile` specification in this repository, that specification also loads a PyTorch model).
+
+
+### Configurable runtime parameters
+
+Although all CLAMS apps are supposed to run as *stateless* HTTP servers, some apps can be configured at request time using [URL query strings](https://en.wikipedia.org/wiki/Query_string). For runtime parameter supported by this app, please visit [CLAMS App Directory](https://apps.clams.ai) and look for the app name and version. 
+
+
+### Running the application
+
+To test the code without running a Flask server use the `test.py` script. 
+
+```bash
+python test.py example-mmif.json out.json
+```
+
+The example MMIF file in `example-mmif.json` depends on there being a video file in `/data/video/`, edit the example file as needed.
+
+To build the Docker image
+
+```bash
+docker build -t app-swt:1.0 -f Containerfile .
+```
+
+To run the container
+
+```bash
+docker run --rm -d -v /Users/Shared/archive/:/data -p 5000:5000 app-swt:1.0
+```
+
+Now you can access the app:
+
+```bash
+curl http://localhost:5000?pretty=true
+curl -X POST -d@input/example-1.mmif http://localhost:5000/
+```
+
+The first gets you the metadata and the second, which may take a while depending on the size of your video file, returns a MMIF object with timeframes added, for example
+
+```json
+{
+  "metadata": {
+    "mmif": "http://mmif.clams.ai/0.4.0"
+  },
+  "documents": [
+    {
+      "@type": "http://mmif.clams.ai/0.4.0/vocabulary/VideoDocument",
+      "properties": {
+        "mime": "video/mpeg",
+        "id": "m1",
+        "location": "file:///data/video/cpb-aacip-690722078b2-shrunk.mp4"
+      }
+    }
+  ],
+  "views": [
+    {
+      "id": "v_0",
+      "metadata": {
+        "timestamp": "2023-11-06T20:00:18.311889",
+        "app": "http://apps.clams.ai/swt-detection",
+        "contains": {
+          "http://mmif.clams.ai/vocabulary/TimeFrame/v1": {
+            "document": "m1"
+          }
+        },
+        "parameters": {
+          "pretty": "True"
+        }
+      },
+      "annotations": [
+        {
+          "@type": "http://mmif.clams.ai/vocabulary/TimeFrame/v1",
+          "properties": {
+            "start": 30000,
+            "end": 40000,
+            "frameType": "slate",
+            "score": 3.909090909090909,
+            "id": "tf_1"
+          }
+        },
+        {
+          "@type": "http://mmif.clams.ai/vocabulary/TimeFrame/v1",
+          "properties": {
+            "start": 56000,
+            "end": 58000,
+            "frameType": "slate",
+            "score": 1.3333333333333333,
+            "id": "tf_2"
+          }
+        }
+      ]
+    }
+  ]
+}
+```

app.py

@@ -0,0 +1,77 @@
+"""
+
+CLAMS app to detect scenes with text.
+
+The kinds of scenes that are recognized include slates, chryons and credits.
+
+"""
+
+import argparse
+import logging
+from typing import Union
+
+from clams import ClamsApp, Restifier
+from mmif import Mmif, View, Annotation, Document, AnnotationTypes, DocumentTypes
+
+import classify
+
+
+logging.basicConfig(filename='swt.log', level=logging.DEBUG)
+
+
+class SwtDetection(ClamsApp):
+
+    def __init__(self):
+        super().__init__()
+
+    def _appmetadata(self):
+        # see https://sdk.clams.ai/autodoc/clams.app.html#clams.app.ClamsApp._load_appmetadata
+        # Also check out ``metadata.py`` in this directory. 
+        # When using the ``metadata.py`` leave this do-nothing "pass" method here. 
+        pass
+
+    def _annotate(self, mmif: Union[str, dict, Mmif], **parameters) -> Mmif:
+        # see https://sdk.clams.ai/autodoc/clams.app.html#clams.app.ClamsApp._annotate
+
+        vds = mmif.get_documents_by_type(DocumentTypes.VideoDocument)
+        if not vds:
+            # TODO: should add warning
+            return mmif
+        vd = vds[0]
+
+        # calculate the frame predictions and extract the timeframes
+        predictions = classify.process_video(vd.location, step=classify.STEP_SIZE)
+        timeframes = classify.extract_timeframes(predictions)
+
+        # aad the timeframes to a new view and return the updated Mmif object
+        new_view: View = mmif.new_view()
+        self.sign_view(new_view, parameters)
+        new_view.new_contain(AnnotationTypes.TimeFrame, document=vd.id)
+        for tf in timeframes:
+            start, end, score, label = tf
+            timeframe_annotation = new_view.new_annotation(AnnotationTypes.TimeFrame)
+            timeframe_annotation.add_property("start", start)
+            timeframe_annotation.add_property("end", end)
+            timeframe_annotation.add_property("frameType", label),
+            timeframe_annotation.add_property("score", score)
+        return mmif
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", action="store", default="5000", help="set port to listen" )
+    parser.add_argument("--production", action="store_true", help="run gunicorn server")
+
+    parsed_args = parser.parse_args()
+
+    app = SwtDetection()
+
+    http_app = Restifier(app, port=int(parsed_args.port))
+    # for running the application in production mode
+    if parsed_args.production:
+        http_app.serve_production()
+    # development mode
+    else:
+        app.logger.setLevel(logging.DEBUG)
+        http_app.run()