brought back prebin configuration and updated see_results script

modeling/__init__.py

@@ -2,10 +2,14 @@
 positive_label = '+'
 
 # full typology from https://github.com/clamsproject/app-swt-detection/issues/1
-FRAME_TYPES = ["B", "S", "W", "L", "O",
-               "M", "I", "N", "E", "P", "Y", "K", "G", "T", "F", "C", "R"]
-FRAME_TYPES_WITH_SUBTYPES = ["B", "SH", "SC", "SD", "SB", "SG", "W", "L", "O",
-                             "M", "I", "N", "E", "P", "Y", "K", "G", "T", "F", "C", "R"]
+FRAME_TYPES = [
+    "B", "S", "I", "C", "R", "M", "O", "W",
+    "N", "Y", "U", "K",
+    "L", "G", "F", "E", "T",
+    "P",
+]
+FRAME_TYPES_WITH_SUBTYPES = FRAME_TYPES.copy() + ['SH', 'SC', 'SD', 'SB', 'SG']
+FRAME_TYPES_WITH_SUBTYPES.remove('S')
 
 # These are time frames that are typically static (that is, the text does not
 # move around or change as with rolling credits). These are frame names after

modeling/classify.py

@@ -20,7 +20,7 @@ def __init__(self, model_stem, logger_name=None):
         model_config_file = f"{model_stem}.yml"
         model_checkpoint = f"{model_stem}.pt"
         model_config = yaml.safe_load(open(model_config_file))
-        self.training_labels = train.pretraining_binned_label(model_config)
+        self.training_labels = train.get_prebinned_labelset(model_config)
         self.featurizer = data_loader.FeatureExtractor(
             img_enc_name=model_config["img_enc_name"],
             pos_length=model_config.get("pos_length", 0),

modeling/gridsearch.py

@@ -119,12 +119,78 @@
     ] + guids_with_challenging_images,  # also block the challenging images
     # {"cpb-aacip-254-75r7szdz"},  # effectively no block except
 ]
-# we no longer use bins, keeping this just for historical reference
-# bins = [{'pre': {'slate': ['S'], 'chyron': ['I', 'N', 'Y'], 'credit': ['C']}}]
+nobinning = {t: t for t in modeling.FRAME_TYPES}
 
-param_keys = ['split_size', 'num_epochs', 'num_layers', 'pos_length', 'pos_unit', 'dropouts', 'img_enc_name', 'pos_abs_th_front', 'pos_abs_th_end', 'pos_vec_coeff', 'block_guids_train', 'block_guids_valid']
+label_bins = {
+    "Bars": ["B"],
+    "Chyron-other": ["Y", "U", "K"],
+    "Chyron-person": ["I", "N"],
+    "Credits": ["C", "R"],
+    "Main": ["M"],
+    "Opening": ["O", "W"],
+    "Slate": ["S", "S:H", "S:C", "S:D", "S:B", "S:G"],
+    "Other-text-sm": ["L", "G", "F", "E", "T"],
+    "Other-text-md": ["M", "O", "W", "L", "G", "F", "E", "T"],
+    "Other-text-lg": ["M", "O", "W", "Y", "U", "K", "L", "G", "F", "E", "T"],
+}
+
+binning_schemes = {
+    "nobinning": nobinning,
+
+    "strict": {
+        "Bars": label_bins["Bars"],
+        "Slate": label_bins["Slate"],
+        "Chyron-person": label_bins["Chyron-person"],
+        "Credits": label_bins["Credits"],
+        "Main": label_bins["Main"],
+        "Opening": label_bins["Opening"],
+        "Chyron-other": label_bins["Chyron-other"],
+        "Other-text": label_bins["Other-text-sm"],
+    },
+
+    "simple": {
+        "Bars": label_bins["Bars"],
+        "Slate": label_bins["Slate"],
+        "Chyron-person": label_bins["Chyron-person"],
+        "Credits": label_bins["Credits"],
+        "Other-text": label_bins["Other-text-lg"],
+    },
+
+    "relaxed": {
+        "Bars": label_bins["Bars"],
+        "Slate": label_bins["Slate"],
+        "Chyron": label_bins["Chyron-other"] + label_bins["Chyron-person"],
+        "Credits": label_bins["Credits"],
+        "Other-text": label_bins["Other-text-md"],
+    },
+
+    "binary-bars": {
+        "Bars": label_bins["Bars"],
+    },
+
+    "binary-slate": {
+        "Slate": label_bins["Slate"],
+    },
+
+    "binary-chyron-strict": {
+        "Chyron-person": label_bins["Chyron-person"],
+    },
+
+    "binary-chyron-relaxed": {
+        "Chyron": label_bins["Chyron-other"] + label_bins["Chyron-person"],
+    },
+
+    "binary-credits": {
+        "Credits": label_bins["Credits"],
+    }
+}
+
+# for single binning configuration, just use the binning dict
+# for multiple binning configurations (for experimental reasons), use the binning scheme names (str)
+prebin = [nobining]
+
+param_keys = ['split_size', 'num_epochs', 'num_layers', 'pos_length', 'pos_unit', 'dropouts', 'img_enc_name', 'pos_abs_th_front', 'pos_abs_th_end', 'pos_vec_coeff', 'block_guids_train', 'block_guids_valid', 'prebin']
 l = locals()
 configs = []
 for vals in itertools.product(*[l[key] for key in param_keys]):
     configs.append(dict(zip(param_keys, vals)))
-

modeling/train.py

@@ -56,12 +56,12 @@ def get_guids(data_dir):
 
 
 def pretraining_bin(label, specs):
-    if specs is None or "bins" not in specs:
+    if specs is None or "prebin" not in specs:
         return int_encode(label)
-    for i, ptbin in enumerate(specs["bins"].values()):
+    for i, ptbin in enumerate(specs["prebin"].values()):
         if label and label in ptbin:
             return i
-    return len(specs["bins"].keys())
+    return len(specs["prebin"].keys())
 
 
 def load_config(config):
@@ -161,11 +161,12 @@ def train(indir, outdir, config_file, configs, train_id=time.strftime("%Y%m%d-%H
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
     # the number of labels (after "pre"-binning)
-    if configs and 'bins' in configs:
-        num_labels = len(configs['bins'].keys()) + 1
+    if configs and 'prebin' in configs:
+        num_labels = len(configs['prebin'].keys()) + 1
     else:
         num_labels = len(FRAME_TYPES) + 1
-
+    labelset = get_prebinned_labelset(configs)
+
     # if split_size > #videos, nothing to "hold-out". Hence, single fold training and validate against the "fixed" set
     if configs['split_size'] >= len(train_all_guids):
         valid_guids = gridsearch.guids_for_fixed_validation_set
@@ -181,7 +182,7 @@ def train(indir, outdir, config_file, configs, train_id=time.strftime("%Y%m%d-%H
             loss, device, train_loader, configs)
         torch.save(model.state_dict(), export_model_file)
         p_config = Path(f'{base_fname}.yml')
-        validate(model, valid_loader, pretraining_binned_label(config), export_fname=f'{base_fname}.csv')
+        validate(model, valid_loader, labelset, export_fname=f'{base_fname}.csv')
         export_train_config(config_file, configs, p_config)
         return
     # otherwise, do k-fold training with k's size = split_size
@@ -206,7 +207,7 @@ def train(indir, outdir, config_file, configs, train_id=time.strftime("%Y%m%d-%H
                 get_net(train.feat_dim, num_labels, configs['num_layers'], configs['dropouts']),
                 loss, device, train_loader, configs)
         torch.save(model.state_dict(), export_model_file)
-        p, r, f = validate(model, valid_loader, pretraining_binned_label(config), export_fname=export_csv_file)
+        p, r, f = validate(model, valid_loader, labelset, export_fname=export_csv_file)
         val_set_spec.append(validation_guids)
         p_scores.append(p)
         r_scores.append(r)
@@ -247,9 +248,9 @@ def export_kfold_results(trial_specs, p_scores, r_scores, f_scores, p_results):
         out.write(f'\trecall = {sum(r_scores) / len(r_scores)}\n')
 
 
-def pretraining_binned_label(config):
-    if 'bins' in config:
-        return list(config["bins"].keys()) + [modeling.negative_label]
+def get_prebinned_labelset(config):
+    if 'prebin' in config:
+        return list(config["prebin"].keys()) + [modeling.negative_label]
     return modeling.FRAME_TYPES + [modeling.negative_label]
 
 
@@ -309,8 +310,13 @@ def train_model(model, loss_fn, device, train_loader, configs):
     for config in configs:
         timestamp = time.strftime("%Y%m%d-%H%M%S")
         backbonename = config['img_enc_name']
+        if isinstance(config['prebin'], str):
+            prebin_name = config['prebin']
+            config['prebin'] = gridsearch.binning_schemes[prebin_name]
+        else:
+            prebin_name = ''
         positionalencoding = "pos" + ("F" if config["pos_vec_coeff"] == 0 else "T")
         train(
             indir=args.indir, outdir=args.outdir, config_file=args.config, configs=config,
-            train_id='.'.join([timestamp, backbonename, positionalencoding])
+            train_id='.'.join(filter(None, [timestamp, backbonename, prebin_name, positionalencoding]))
         )