Experiments with evaluation and visualization

evaluate.py

@@ -0,0 +1,78 @@
+"""Timepoint evaluation
+
+We have the results of SWT plus annotations files:
+
+- MMIF files with annotations for all time frames with within the timeframes
+  time points every 1000 milliseconds
+- GBH annotations with approcimately a timepoint for every 2 seconds, but not
+  quite, so the annotations and the data do not line up perfectly
+
+Because of the mismatch of system predictions and gold predictions we need to do
+one of two things:
+
+- Find an approximate gold timepoint for each timepoint predicted by the system.
+- Run the code on the exact timepoints, this would need to be done 
+
+Since the timepoints never seem to be more out of sync that 32ms, we will go with
+the first.
+
+"""
+
+from mmif import Mmif
+
+
+# MMIF files
+mmif_files = ('ex-aapb-50.json', 'ex-aapb-69.json', 'ex-aapb-75.json')
+
+# Annotation files
+gold_files = (
+    'modeling/annotations-gbh/cpb-aacip-507-028pc2tp2z.csv',
+    'modeling/annotations-gbh/cpb-aacip-690722078b2.csv',
+    'modeling/annotations-gbh/cpb-aacip-75-72b8h82x.csv')
+
+bin_mappings = {
+    'B': 'bars',
+    'S': 'slate', 'S:H': 'slate', 'S:C': 'slate', 'S:D': 'slate', 'S:G': 'slate',
+    'I': 'chyron', 'N': 'chyron', 'Y': 'chyron',
+    'C': 'credits',}
+
+
+fh = open('collected_different.txt', 'w')
+
+
+def evaluate(mmif_file: str, gold_file: str):
+    read_mmif_points(mmif_file)
+
+
+def read_mmif_points(mmif_file: str):
+    mmif_obj = Mmif(open(mmif_file).read())
+    for view in mmif_obj.views:
+        annotations = view.annotations
+        break
+    timepoints_idx = {}
+    timeframes = []
+    print(mmif_file, len(annotations))
+    for annotation in annotations:
+        if 'TimeFrame' in str(annotation.at_type):
+            timeframes.append(annotation)
+        elif 'TimePoint' in str(annotation.at_type):
+            timepoints_idx[annotation.id] = annotation
+    for tf in timeframes:
+        post_label = tf.get_property('frameType')
+        #print(tf.id, post_label)
+        for tp_id in tf.get_property('targets'):
+            tp = timepoints_idx[tp_id]
+            t = tp.get_property('timePont')
+            pre_label = tp.get_property('label')
+            pre_label = bin_mappings.get(pre_label, 'other')
+            if pre_label != post_label:
+                fh.write(f'{t} {pre_label} {post_label}\n')
+            print(t, pre_label, post_label)
+
+
+
+if __name__ == '__main__':
+
+    data = zip(mmif_files, gold_files)
+    for mmif_file, gold_file in data:
+        evaluate(mmif_file, gold_file)

modeling/classify.py

@@ -76,7 +76,7 @@ def __str__(self):
         return (f"<Classifier "
                 + f'img_enc_name="{self.model_config["img_enc_name"]}" '
                 + f'pos_enc_name="{self.model_config["pos_enc_name"]}" '
-                + f'sample_rate={self.get_sample_rate()}>')
+                + f'sample_rate={self.sample_rate}>')
 
     def process_video(self, mp4_file: str) -> list:
         """Loops over the frames in a video and for each frame extracts the features
@@ -111,12 +111,6 @@ def process_video(self, mp4_file: str) -> list:
             predictions.append(prediction)
         return predictions
 
-    def get_sample_rate(self) -> int:
-        try:
-            return self.sample_rate
-        except AttributeError:
-            return None
-
     def pp(self):
         # debugging method
         print(f"Classifier {self.model_file}")

visualize.py

@@ -4,6 +4,10 @@
 
 Usage:
 
+$ python visualize.py -m <MMIF_FILE>
+
+    MMIF_FILE - An MMIF file that refers to a local MP4 video vile
+
 $ python visualize.py -v <VIDEO_FILE> -p <PREDICTIONS_FILE>
 
     VIDEO_FILE        -  an MP4 video file
@@ -25,12 +29,14 @@
 
 import os, json, argparse
 import cv2
-
+from mmif import Mmif, DocumentTypes
 import modeling
 
+
 # Edit this if we use different labels
 LABELS = ('slate', 'chyron', 'credits')
 LABELS = ('bars', 'slate', 'chyron', 'credits', 'copy', 'text', 'person')
+LABELS = ('bars', 'slate', 'chyron', 'credits')
 
 
 STYLESHEET = '''
@@ -122,19 +128,35 @@ def get_color_class(score: float):
 if __name__ == '__main__':
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("-v", metavar='FILENAME', required=True, help="video file")
-    parser.add_argument("-p", metavar='FILENAME', required=True, help="predictions file")
+    parser.add_argument("-m", metavar='FILENAME', help="MMIF file")
+    parser.add_argument("-v", metavar='FILENAME', help="video file")
+    parser.add_argument("-p", metavar='FILENAME', help="predictions file")
     args = parser.parse_args()
 
-    video_file = args.v
-    predictions_file = args.p
+    if args.m:
+
+        # Read the MMIF file and get the video location
+        mmif_obj = Mmif(open(args.m).read())
+        locations = mmif_obj.get_documents_locations(DocumentTypes.VideoDocument)
+        video_location = locations[0]
+        print(video_location)
+        # Get the predictions
+        # Prediction: a pair (milliseconds, scores), scores: a list of probabilities
+        # Read them from all the timepoints in the timeframes
+        for view in mmif_obj.views:
+            for annotation in view.annotations:
+                if 'TimeFrame' in str(annotation.at_type):
+                    print(annotation.properties.get('frameType'))
 
-    basename = os.path.splitext(os.path.basename(predictions_file))[0]
-    outdir = os.path.join('html', basename)
-    outdir_frames = os.path.join(outdir, 'frames')
-    index_file = os.path.join(outdir, f'index-{"-".join(LABELS)}.html')
-    os.makedirs(outdir_frames, exist_ok=True)
 
-    predictions = load_predictions(predictions_file)
-    #create_frames(video_file, predictions, outdir_frames)
-    visualize_predictions(predictions, LABELS, index_file, video_file)
+    else:
+        video_file = args.v
+        predictions_file = args.p
+        basename = os.path.splitext(os.path.basename(predictions_file))[0]
+        outdir = os.path.join('html', basename)
+        outdir_frames = os.path.join(outdir, 'frames')
+        index_file = os.path.join(outdir, f'index-{"-".join(LABELS)}.html')
+        os.makedirs(outdir_frames, exist_ok=True)
+        predictions = load_predictions(predictions_file)
+        #create_frames(video_file, predictions, outdir_frames)
+        visualize_predictions(predictions, LABELS, index_file, video_file)