forked from IsmaelCesar/SniffNets
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathevaluation.py
148 lines (123 loc) · 6.04 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
"""
Module contains the implementation of some
model evaluation procedure.
"""
import os
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
def calculate_mean_and_stdd(list_of_values):
"""
Calculates the mean and standard deviation of a set
:param list_of_values:
:return: mean and standard deviation
"""
mean = 0
stdd = 0
set_size = len(list_of_values)
for acc in list_of_values:
mean += acc
mean = mean/set_size
for acc in list_of_values:
stdd += np.power(acc - mean, 2)
stdd = np.sqrt(stdd/set_size)
return mean, stdd
def save_results_into_filesystem_windows(experiments_folder, model_folder, super_exp_folder, sub_exp_folder,
window_file, H, test_data, test_labels, batch_size, model):
if not os.path.exists(experiments_folder+model_folder):
os.mkdir(experiments_folder+model_folder)
if not os.path.exists(super_exp_folder):
os.mkdir(super_exp_folder)
if not os.path.exists(sub_exp_folder):
os.mkdir(sub_exp_folder)
# if window_size != "":
# os.mkdir(sub_exp_folder+"window_"+window_size+"/")
train_mean_acc, train_stdd_acc = calculate_mean_and_stdd(H.history["accuracy"])
val_mean_acc, val_stdd_acc = calculate_mean_and_stdd(H.history["val_accuracy"])
with open(window_file + "eval.txt", 'w') as f:
predictions = model.predict(test_data, batch_size=batch_size)
value = classification_report(test_labels.argmax(axis=1),
predictions.argmax(axis=1))
value += "\nTrain acc mean: "+str(train_mean_acc)+"\t ,Train acc stdd: "+str(train_stdd_acc)
value += ("\nValidation acc mean: " + str(val_mean_acc) +
"\t ,Validation acc stdd: " + str(val_stdd_acc) + "\n\n")
print(value)
f.write(value)
f.close()
def evaluate_model_windows(test_data, test_labels, batch_size, model, n_epochs, H, experiments_folder,
dataset_name, sub_dataset_name, model_folder, window_size="", save_results=False):
# Evaluating model
print("[INFO] Evaluating Network")
super_exp_folder = experiments_folder + model_folder + dataset_name
sub_exp_folder = experiments_folder + model_folder + dataset_name + sub_dataset_name
window_file = sub_exp_folder+"window_"+window_size+"_"
if save_results:
save_results_into_filesystem_windows(experiments_folder, model_folder, super_exp_folder,
sub_exp_folder, window_file, H, test_data, test_labels,
batch_size, model)
plt.style.use("ggplot")
plt.figure()
plt.plot(np.arange(0, n_epochs), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, n_epochs), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, n_epochs), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, n_epochs), H.history["val_accuracy"], label="val_acc")
if window_size != "":
plt.title("Training Loss and Accuracy Window "+window_size)
else:
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend()
plt.show()
if save_results:
plt.savefig(window_file + "LossAccComparison.png")
plt.close('all')
# plt.show()
def save_results_into_filesystem(experiments_folder, model_folder, super_exp_folder, sub_exp_folder,
H, test_data, test_labels, batch_size, model):
if not os.path.exists(experiments_folder + model_folder):
os.mkdir(experiments_folder + model_folder)
if not os.path.exists(super_exp_folder):
os.mkdir(super_exp_folder)
if not os.path.exists(sub_exp_folder):
os.mkdir(sub_exp_folder)
# if window_size != "":
# os.mkdir(sub_exp_folder+"window_"+window_size+"/")
train_mean_acc, train_stdd_acc = calculate_mean_and_stdd(H.history["accuracy"])
val_mean_acc, val_stdd_acc = calculate_mean_and_stdd(H.history["val_accuracy"])
with open("eval.txt", 'w') as f:
predictions = model.predict(test_data, batch_size=batch_size)
value = classification_report(test_labels.argmax(axis=1),
predictions.argmax(axis=1))
value += "\nTrain acc mean: " + str(train_mean_acc) + "\t ,Train acc stdd: " + str(train_stdd_acc)
value += ("\nValidation acc mean: " + str(val_mean_acc) +
"\t ,Validation acc stdd: " + str(val_stdd_acc) + "\n\n")
print(value)
f.write(value)
f.close()
return os.path.join(experiments_folder + model_folder, super_exp_folder, sub_exp_folder)
def evaluate_model(test_data, test_labels, batch_size, model, n_epochs, H, experiments_folder,
dataset_name, sub_dataset_name, model_folder, save_results=False):
# Evaluating model
print("[INFO] Evaluating Network")
super_exp_folder = experiments_folder + model_folder + dataset_name
sub_exp_folder = experiments_folder + model_folder + dataset_name + sub_dataset_name
path_to_exp_folder = None
if save_results:
path_to_exp_folder = save_results_into_filesystem(experiments_folder, model_folder, super_exp_folder,
sub_exp_folder, H, test_data, test_labels,
batch_size, model)
plt.style.use("ggplot")
plt.subplots()
plt.plot(np.arange(0, n_epochs), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, n_epochs), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, n_epochs), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, n_epochs), H.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend()
plt.show()
if save_results:
plt.savefig(os.path.join(path_to_exp_folder, "LossAccComparison.png"))
plt.close('all')