-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathevaluate.py
140 lines (120 loc) · 4.44 KB
/
evaluate.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
import numpy as np
import pandas as pd
import os
import csv
import argparse
from tqdm import tqdm
from PIL import Image
'''
Script for evaluation of output
'''
def evaluate_images(dst_path, img_pred, img_true, step=0.01):
'''
experimental purpose (this method is not used)
'''
thres = 0.0
best_f = 0.0
best_thres = 0.0
while thres < 1.0:
img_thred = (img_pred > thres).astype(np.int32)
try:
precision = np.sum(np.logical_and(img_thred, img_true)) / np.sum(img_thred)
recall = np.sum(np.logical_and(img_thred, img_true)) / np.sum(img_true)
f = 2*precision*recall / (recall + precision)
except ZeroDivisionError:
f = 0.0
if f > best_f:
best_f = f
best_thres = thres
thres += step
return best_f, best_thres
def write_evaluation(dst_dir, img_pred, img_true, step=0.01):
'''
make the temporary file for get_ods_ap().
This method automatically change the threshold to calculate the ODS and AP.
args:
dst_dir (str): location to store temporary files
img_pred (ndarray): a predicted image
img_true (ndarray): a ground truth image
step (float): step of threshold
'''
pred_csv = open(os.path.join(dst_dir, "num_pred.csv"), 'a')
true_csv = open(os.path.join(dst_dir, "num_true.csv"), 'a')
and_csv = open(os.path.join(dst_dir, "num_and.csv"), 'a')
writer_pred = csv.writer(pred_csv)
writer_true = csv.writer(true_csv)
writer_and = csv.writer(and_csv)
thres = 0.0
list_pred = []
list_and = []
num_true = [np.sum(img_true)]
while thres < 1.0:
img_thred = (img_pred > thres).astype(np.int32)
num_pred = np.sum(img_thred)
num_and = np.sum(np.logical_and(img_thred, img_true))
list_pred.append(num_pred)
list_and.append(num_and)
thres += step
writer_pred.writerow(list_pred)
writer_and.writerow(list_and)
writer_true.writerow(num_true)
pred_csv.close()
true_csv.close()
and_csv.close()
def get_ods_ap(src_dir):
'''
Calculate the ODS(optimal dataset scale) and AP(average precision).
You must run write_evaluation before this function.
args:
src_dir (str): location of temporary files
'''
df_pred = pd.read_csv(os.path.join(src_dir, "num_pred.csv"), header=None)
df_true = pd.read_csv(os.path.join(src_dir, "num_true.csv"), header=None)
df_and = pd.read_csv(os.path.join(src_dir, "num_and.csv"), header=None)
if not len(df_pred) == len(df_true) == len(df_and):
print("Different rows")
return
n_cols = len(df_pred.columns)
f_best = 0.0
sum_precision = 0.0
for i in range(n_cols):
try:
precision = np.sum(df_and.iloc[:,i]) / np.sum(df_pred.iloc[:,i])
recall = np.sum(df_and.iloc[:,i]) / np.sum(df_true.iloc[:])
f = float(2*precision*recall / (recall + precision))
sum_precision += precision
except ZeroDivisionError:
f = 0.0
if f > f_best:
f_best = f
ois = f_best
ap = sum_precision / n_cols
return ois, ap
def main(args):
if not os.path.exists(args.dst):
os.makedirs(args.dst)
with open(os.path.join(dst_dir, "num_pred.csv"), 'w') as f:
pass
with open(os.path.join(dst_dir, "num_true.csv"), 'w') as f:
pass
with open(os.path.join(dst_dir, "num_and.csv"), 'w') as f:
pass
files = os.listdir(args.preddir)
for f in tqdm(files):
pred_path = os.path.join(args.preddir, f)
true_path = os.path.join(args.truedir, f)
img_pred = np.asarray(Image.open(pred_path)) / 255.0
img_true = np.asarray(Image.open(true_path)) / 255.0
write_evaluation(args.dst, img_pred, img_true)
ods, ap = get_ods_ap(args.dst)
with open(os.path.join(args.dst, "result.txt"), 'w') as f:
text = "ods:{0}, ap:{1}".format(ods, ap)
f.write(text)
print(text)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--preddir", required=True, help="specify the directory of predicted images")
parser.add_argument("--truedir", required=True, help="specify the directory of true images")
parser.add_argument("--dst", required=True, help="specifiy destination directory path")
args = parser.parse_args()
main(args)