New Experiments

mlp_regr_every_eval.py

@@ -22,7 +22,7 @@
 
     model_dir = opt.save_path[:-14]
     model_folders = os.listdir(model_dir)
-
+    
     for folder in tqdm(model_folders):
         cur_model = tf.keras.models.load_model(model_dir + folder)
         cur_epoch = folder[-4:]

mlp_regr_every_train.py

@@ -26,7 +26,7 @@
         verbose=1,
         save_best_only=True,
         save_weights_only=False,
-        save_freq='epoch', # Save model for every 5 epoch
+        save_freq='epoch', # Save model for every epoch
         mode='auto'
     )
 

mlp_regr_new.py

@@ -10,10 +10,10 @@
     x_train_data, y_train_data = utils.import_data(
         data_root_path=opt.data_root_path,
         selected_bands=opt.selected_bands,
-        train_for_regression=opt.regression,
+        train_for_regression=True,
         derivative=opt.derivative,
         mushroom_class=opt.mushroom_class,
-        normalize="zscore",
+        normalize='zscore',
         shuffle=True,
     )
 
@@ -26,31 +26,21 @@
         train_for_regression=False,
         derivative=opt.derivative,
         mushroom_class=opt.mushroom_class,
-        normalize="zscore",
+        normalize='zscore',
         shuffle=False,
     )
 
-    # Callbacks
-    checkpoint = ModelCheckpoint(
-        opt.save_path,
-        monitor='val_mean_squared_error',
-        verbose=2,
-        save_best_only=True,
-        save_weights_only=False,
-        mode='auto',
-        initial_value_threshold=0.0
-    )
-
     early_stop = EarlyStopping(
         monitor='val_mean_squared_error',
         patience=500,
         restore_best_weights=True,
         verbose=0
     )
 
-    callbacks = [checkpoint, early_stop]
+    callbacks = [early_stop]
 
-    layout = [512]
+    model_id = "MLP32"
+    layout = opt.mlp_layout[model_id]
 
     model = trainer.build_tf_model(
         neurons_layout=layout,
@@ -70,4 +60,4 @@
     )
 
     preds = model.predict(x_all_data)
-    trainer.eval_mlp_regression(preds, opt.mushroom_class, layout)
+    trainer.eval_mlp_regression(preds, opt.mushroom_class, model_id)

mlp_vs_svm.py

@@ -31,35 +31,13 @@
 
     callbacks = [early_stop]
 
-    mlp_layouts = {
-        'MLP21': [64, 32],
-        'MLP22': [512, 512],
-        'MLP31': [128, 64, 32],
-        'MLP32': [512, 512, 512],
-        'MLP41': [256, 128, 64, 32],
-        'MLP42': [512, 512, 512, 512],
-        'MLP51': [512, 256, 128, 64, 32],
-        'MLP52': [512, 512, 512, 512, 512]
-    }
+    mlp_layouts = opt.mlp_layout
     # mlp_layout_acc_dict has the same keys as mlp_layouts, but the value is a list,
     # as the list contains different accuracy from K-Fold.
     # e.g., {'MLP21': [20.0, 34.0, 15.0]}
     mlp_layout_acc_dict = {}
 
-    svm_layouts = {
-        'lSVM1': [0.01],
-        'lSVM2': [1],
-        'lSVM3': [100],
-        'kSVM1': [0.01, 0.001],
-        'kSVM2': [0.01, 'scale'],
-        'kSVM3': [0.01, 'auto'],
-        'kSVM4': [1, 0.001],
-        'kSVM5': [1, 'scale'],
-        'kSVM6': [1, 'auto'],
-        'kSVM7': [100, 0.001],
-        'kSVM8': [100, 'scale'],
-        'kSVM9': [100, 'auto']
-    }
+    svm_layouts = opt.svm_layouts
     # svm_layout_acc_dict has the same keys as svm_layouts, but the value is a list,
     # as the list contains different accuracy from K-Fold.
     # e.g., {'lSVM1': [20.0, 34.0, 15.0]}

svr_regr.py

@@ -0,0 +1,54 @@
+# Custom Modules
+from sklearn.svm import SVR
+import utils
+import trainer
+
+if __name__ == "__main__":
+    opt = utils.Config()
+
+    # Prepare data and train_test_split
+    x_train_data, y_train_data = utils.import_data(
+        data_root_path=opt.data_root_path,
+        selected_bands=opt.selected_bands,
+        train_for_regression=opt.regression,
+        derivative=opt.derivative,
+        mushroom_class=opt.mushroom_class,
+        normalize="zscore",
+        shuffle=True,
+    )
+
+    # Warning from sklearn documentation:
+    # If training data is not C-contihuous, the error may occur.
+    print(
+        f"Check if the input data is C-contiguous: {x_train_data.flags['C_CONTIGUOUS']}")
+
+    # Import evaluation data
+    x_all_data, y_all_data = utils.import_data(
+        data_root_path=opt.data_root_path,
+        selected_bands=opt.selected_bands,
+        train_for_regression=False,
+        derivative=opt.derivative,
+        mushroom_class=opt.mushroom_class,
+        normalize="zscore",
+        shuffle=False,
+    )
+
+    model_id = "lSVM3"
+    params = opt.svm_layout[model_id]
+
+    if len(params) == 1:
+        model = SVR(
+            C=params[0],
+            kernel='linear'
+        )
+    else:
+        model = SVR(
+            C=params[0],
+            gamma=params[1],
+            kernel='rbf'
+        )
+
+    model.fit(x_train_data, y_train_data)
+    preds = model.predict(x_all_data)
+    trainer.eval_regression(opt.mushroom_class, preds, model_id)
+

trainer.py

@@ -166,9 +166,9 @@ def build_tf_model(
         model.add(Dense(256, activation=activation))
         model.add(Dense(256, activation=activation))
         model.add(Dense(256, activation=activation))
-        model.add(Dropout(0.3))
+        # model.add(Dropout(0.3))
         model.add(Dense(256, activation=activation))
-        model.add(Dropout(0.3))
+        # model.add(Dropout(0.3))
     else:
         for neurons in neurons_layout:
             model.add(Dense(neurons, activation=activation))
@@ -195,7 +195,6 @@ def build_tf_model(
 def eval_mlp_regression(preds, mushroom_class, layout):
     mean_regression_scores = _mean_pred_per_class(preds, mushroom_class)
 
-
     ticks = [f"D{i}" for i in range(0, 30, 2)]
     if mushroom_class == "A":
         x_spaces = [i for i in range(0, 14)]
@@ -212,7 +211,8 @@ def eval_mlp_regression(preds, mushroom_class, layout):
     plt.ylabel("Freshness Decay Score")
     plt.ylim(-.2, 1.2)
     plt.xticks(x_spaces, ticks)
-    plt.savefig(f"./mlp_regr_origin_results_{mushroom_class}/{mushroom_class}_{layout}.png")
+    # plt.savefig(f"./mlp_regr_origin_results_{mushroom_class}/{mushroom_class}_{layout}.png")
+    plt.savefig(f"./regr_curves/{mushroom_class}_{layout}.png")
 
 
 def _mean_pred_per_class(preds, mushroom_class):
@@ -238,28 +238,33 @@ def build_SVR(kernel=None, gamma='auto', C=None):
     return model
 
 
-def eval_regression(mushroom_class, preds, C, gamma=None):
+def eval_regression(mushroom_class, preds, model_id):
     '''Evaluate regression model'''
-    mean_regression_scores = _mean_pred_per_class(preds)
-
-    plot_freshness_curve(mushroom_class, C, gamma, mean_regression_scores)
+    mean_regression_scores = _mean_pred_per_class(preds, mushroom_class)
 
-    print(mean_regression_scores)
+    plot_freshness_curve(mushroom_class, mean_regression_scores, model_id)
 
 
-def plot_freshness_curve(mushroom_class, C, gamma, mean_regression_scores):
-    plt.figure()
-    if gamma == None:
-        kernel = 'Linear'
-        plt.title(f"kernel: {kernel}, C = {C}")
-        plt.plot(mean_regression_scores)
-        plt.savefig(f"../svr_results/{mushroom_class}_{kernel}_{C}.png")
+def plot_freshness_curve(mushroom_class, mean_regression_scores, model_id):
+    ticks = [f"D{i}" for i in range(0, 30, 2)]
+    if mushroom_class == "A":
+        x_spaces = [i for i in range(0, 14)]
+        ticks.pop(8)
     else:
-        kernel = 'RBF'
-        plt.plot(mean_regression_scores)
-        plt.title(f"kernel: {kernel}, gamma = {gamma}, C = {C}")
-        plt.savefig(
-            f"../svr_results/{mushroom_class}_{kernel}_{gamma}_{C}.png")
+        x_spaces = [i for i in range(0, 15)]
+
+    plt.figure()
+    plt.plot(x_spaces, mean_regression_scores, marker="o")
+    plt.axhline(y=0.0, linestyle='--', color='r')
+    plt.axhline(y=1.0, linestyle='--', color='r')
+    plt.xlabel("Days")
+    plt.ylabel("Freshness Decay Score")
+    plt.ylim(-.2, 1.2)
+    plt.xticks(x_spaces, ticks)
+    plt.title(
+        f"SVR freshness curve with {model_id} on class {mushroom_class}.")
+    plt.savefig(
+        f"./regr_curves/{mushroom_class}_{model_id}.png")
 
 
 if __name__ == "__main__":

utils.py

@@ -16,17 +16,42 @@ class Config():
     def __init__(self):
         # Data parameters
         self.data_root_path = '../data/'
-        self.derivative = False
+        self.derivative = True
         if self.derivative:
             self.selected_bands = [i for i in range(299)]
         else:
             self.selected_bands = [i for i in range(300)]
-        self.regression = False
-        # self.save_path = "./original_regr_A/cp-{epoch:04d}"
-        self.save_path = "MLP_regression_B_relu.hdf5"
-        self.mushroom_class = "B"
+        self.regression = True
+        self.save_path = "./original_regr_A/cp-{epoch:04d}"
+        # self.save_path = "MLP_regression_A_relu.hdf5"
+        self.mushroom_class = "A"
         self.train_ratio = 0.8  # 0.8 for NN, 0.5 for SVM.
 
+        self.mlp_layout = {
+            'MLP21': [64, 32],
+            'MLP22': [512, 512],
+            'MLP31': [128, 64, 32],
+            'MLP32': [512, 512, 512],
+            'MLP41': [256, 128, 64, 32],
+            'MLP42': [512, 512, 512, 512],
+            'MLP51': [512, 256, 128, 64, 32],
+            'MLP52': [512, 512, 512, 512, 512]
+        }
+        self.svm_layout = {
+            'lSVM1': [0.01],
+            'lSVM2': [1],
+            'lSVM3': [100],
+            'kSVM1': [0.01, 0.001],
+            'kSVM2': [0.01, 'scale'],
+            'kSVM3': [0.01, 'auto'],
+            'kSVM4': [1, 0.001],
+            'kSVM5': [1, 'scale'],
+            'kSVM6': [1, 'auto'],
+            'kSVM7': [100, 0.001],
+            'kSVM8': [100, 'scale'],
+            'kSVM9': [100, 'auto']
+        }
+
         # Model parameters
         self.n_hidden_layers = 1
         self.activation = 'relu'