From 5590a3e975afa9008687a8533aa167bebb35dc9f Mon Sep 17 00:00:00 2001 From: Marc Verhagen Date: Fri, 9 Feb 2024 12:24:30 -0500 Subject: [PATCH] Experiments with evaluation and visualization --- evaluate.py | 78 ++++++++++++++++++++++++++++++++++++++++++++ modeling/classify.py | 8 +---- visualize.py | 48 +++++++++++++++++++-------- 3 files changed, 114 insertions(+), 20 deletions(-) create mode 100644 evaluate.py diff --git a/evaluate.py b/evaluate.py new file mode 100644 index 0000000..36b40d4 --- /dev/null +++ b/evaluate.py @@ -0,0 +1,78 @@ +"""Timepoint evaluation + +We have the results of SWT plus annotations files: + +- MMIF files with annotations for all time frames with within the timeframes + time points every 1000 milliseconds +- GBH annotations with approcimately a timepoint for every 2 seconds, but not + quite, so the annotations and the data do not line up perfectly + +Because of the mismatch of system predictions and gold predictions we need to do +one of two things: + +- Find an approximate gold timepoint for each timepoint predicted by the system. +- Run the code on the exact timepoints, this would need to be done + +Since the timepoints never seem to be more out of sync that 32ms, we will go with +the first. + +""" + +from mmif import Mmif + + +# MMIF files +mmif_files = ('ex-aapb-50.json', 'ex-aapb-69.json', 'ex-aapb-75.json') + +# Annotation files +gold_files = ( + 'modeling/annotations-gbh/cpb-aacip-507-028pc2tp2z.csv', + 'modeling/annotations-gbh/cpb-aacip-690722078b2.csv', + 'modeling/annotations-gbh/cpb-aacip-75-72b8h82x.csv') + +bin_mappings = { + 'B': 'bars', + 'S': 'slate', 'S:H': 'slate', 'S:C': 'slate', 'S:D': 'slate', 'S:G': 'slate', + 'I': 'chyron', 'N': 'chyron', 'Y': 'chyron', + 'C': 'credits',} + + +fh = open('collected_different.txt', 'w') + + +def evaluate(mmif_file: str, gold_file: str): + read_mmif_points(mmif_file) + + +def read_mmif_points(mmif_file: str): + mmif_obj = Mmif(open(mmif_file).read()) + for view in mmif_obj.views: + annotations = view.annotations + break + timepoints_idx = {} + timeframes = [] + print(mmif_file, len(annotations)) + for annotation in annotations: + if 'TimeFrame' in str(annotation.at_type): + timeframes.append(annotation) + elif 'TimePoint' in str(annotation.at_type): + timepoints_idx[annotation.id] = annotation + for tf in timeframes: + post_label = tf.get_property('frameType') + #print(tf.id, post_label) + for tp_id in tf.get_property('targets'): + tp = timepoints_idx[tp_id] + t = tp.get_property('timePont') + pre_label = tp.get_property('label') + pre_label = bin_mappings.get(pre_label, 'other') + if pre_label != post_label: + fh.write(f'{t} {pre_label} {post_label}\n') + print(t, pre_label, post_label) + + + +if __name__ == '__main__': + + data = zip(mmif_files, gold_files) + for mmif_file, gold_file in data: + evaluate(mmif_file, gold_file) diff --git a/modeling/classify.py b/modeling/classify.py index 0b66a29..7a97a42 100644 --- a/modeling/classify.py +++ b/modeling/classify.py @@ -76,7 +76,7 @@ def __str__(self): return (f"') + + f'sample_rate={self.sample_rate}>') def process_video(self, mp4_file: str) -> list: """Loops over the frames in a video and for each frame extracts the features @@ -111,12 +111,6 @@ def process_video(self, mp4_file: str) -> list: predictions.append(prediction) return predictions - def get_sample_rate(self) -> int: - try: - return self.sample_rate - except AttributeError: - return None - def pp(self): # debugging method print(f"Classifier {self.model_file}") diff --git a/visualize.py b/visualize.py index 1d61d97..3346c42 100644 --- a/visualize.py +++ b/visualize.py @@ -4,6 +4,10 @@ Usage: +$ python visualize.py -m + + MMIF_FILE - An MMIF file that refers to a local MP4 video vile + $ python visualize.py -v -p VIDEO_FILE - an MP4 video file @@ -25,12 +29,14 @@ import os, json, argparse import cv2 - +from mmif import Mmif, DocumentTypes import modeling + # Edit this if we use different labels LABELS = ('slate', 'chyron', 'credits') LABELS = ('bars', 'slate', 'chyron', 'credits', 'copy', 'text', 'person') +LABELS = ('bars', 'slate', 'chyron', 'credits') STYLESHEET = ''' @@ -122,19 +128,35 @@ def get_color_class(score: float): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("-v", metavar='FILENAME', required=True, help="video file") - parser.add_argument("-p", metavar='FILENAME', required=True, help="predictions file") + parser.add_argument("-m", metavar='FILENAME', help="MMIF file") + parser.add_argument("-v", metavar='FILENAME', help="video file") + parser.add_argument("-p", metavar='FILENAME', help="predictions file") args = parser.parse_args() - video_file = args.v - predictions_file = args.p + if args.m: + + # Read the MMIF file and get the video location + mmif_obj = Mmif(open(args.m).read()) + locations = mmif_obj.get_documents_locations(DocumentTypes.VideoDocument) + video_location = locations[0] + print(video_location) + # Get the predictions + # Prediction: a pair (milliseconds, scores), scores: a list of probabilities + # Read them from all the timepoints in the timeframes + for view in mmif_obj.views: + for annotation in view.annotations: + if 'TimeFrame' in str(annotation.at_type): + print(annotation.properties.get('frameType')) - basename = os.path.splitext(os.path.basename(predictions_file))[0] - outdir = os.path.join('html', basename) - outdir_frames = os.path.join(outdir, 'frames') - index_file = os.path.join(outdir, f'index-{"-".join(LABELS)}.html') - os.makedirs(outdir_frames, exist_ok=True) - predictions = load_predictions(predictions_file) - #create_frames(video_file, predictions, outdir_frames) - visualize_predictions(predictions, LABELS, index_file, video_file) + else: + video_file = args.v + predictions_file = args.p + basename = os.path.splitext(os.path.basename(predictions_file))[0] + outdir = os.path.join('html', basename) + outdir_frames = os.path.join(outdir, 'frames') + index_file = os.path.join(outdir, f'index-{"-".join(LABELS)}.html') + os.makedirs(outdir_frames, exist_ok=True) + predictions = load_predictions(predictions_file) + #create_frames(video_file, predictions, outdir_frames) + visualize_predictions(predictions, LABELS, index_file, video_file)