forked from xinyadu/grit_doc_event_entity
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_cases_eval.py
126 lines (103 loc) · 6.61 KB
/
test_cases_eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
from eval import eval_ceaf
if __name__ == "__main__":
print("================= case 1 (in the paper) =================")
golds = {
"docid1": {"Target": [["Pilmai telephone company building", "telephone company building", "telephone company offices"], ["water pipes"], ["public telephone booth"]]},
}
preds = {
"docid1": {"Target": [["water pipes"], ["Pilmai telephone company building"], ["public telephone booth"], ["telephone company offices"]]},
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print
print("================= case 2 (in the paper) =================")
golds = {
"docid1": {"Target": [["Pilmai telephone company building", "telephone company building", "telephone company offices"], ["water pipes"], ["public telephone booth"]]},
}
preds = {
"docid1": {"Target": [["Pilmai telephone company building"], ["water pipes"], ["public telephone booth"]]},
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print
print("================= case 3 (in the paper) =================")
golds = {
"docid1": {"Target": [["Pilmai telephone company building", "telephone company building", "telephone company offices"], ["water pipes"], ["public telephone booth"]]},
}
preds = {
"docid1": {"Target": [["Pilmai telephone company building"], ["public telephone booth"]]},
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print
print("\n\n================= case 4 =================")
golds = {
"docid1": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]},
}
preds = {
"docid1": {"PerpInd": [["m4"], ["m1"], ["m5"], ["m6"], ["m3"]]},
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print
print("================= case 5 =================")
golds = {
"docid2": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]},
}
preds = {
"docid2": {"PerpInd": [["m1", "m2"], ["m4"], ["m5"], ["m6"]]},
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print
print("================= case 6 =================")
golds = {
"docid3": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]},
}
preds = {
"docid3": {"PerpInd": [["m1", "m2", "m3", "m4"], ["m5"], ["m6"]]},
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print
print("================= case 7 =================")
golds = {
"docid1": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]},
"docid2": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]},
"docid3": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]},
"docid4": {"PerpInd": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]],
"PerpOrg": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]],
"Target": [["m1", "m2", "m3"], ["m4"], ["m5"], ["m6"]]}
}
preds = {
"docid1": {"PerpInd": [["m4"], ["m1"], ["m5"], ["m6"], ["m3"]]},
"docid2": {"PerpInd": [["m1", "m2"], ["m4"], ["m5"], ["m6"]]},
"docid3": {"PerpInd": [["m1", "m2", "m3", "m4"], ["m5"], ["m6"]]},
"docid4": {"PerpInd": [["m4"], ["m1"], ["m5"], ["m6"], ["m3"]],
"PerpOrg": [["m1", "m2"], ["m4"], ["m5"], ["m6"]],
"Target": [["m1", "m2", "m3", "m4"], ["m5"], ["m6"]]}
}
results = eval_ceaf(preds, golds, docids=[])
print("golds", golds)
print("preds", preds)
print("phi_strict: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["strict"]["micro_avg"]["p"] * 100, results["strict"]["micro_avg"]["r"] * 100, results["strict"]["micro_avg"]["f1"] * 100))
print("phi_prop: P: {:.2f}%, R: {:.2f}%, F1: {:.2f}%".format(results["prop"]["micro_avg"]["p"] * 100, results["prop"]["micro_avg"]["r"] * 100, results["prop"]["micro_avg"]["f1"] * 100))
print