-
Notifications
You must be signed in to change notification settings - Fork 1
/
eval.py
90 lines (81 loc) · 4.72 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
import pickle
import os
import numpy as np
from sklearn.metrics import roc_auc_score, confusion_matrix
import sys
from utils import scorebinary, anomap
import cv2
def eval_p(itr, dataset, predict_dict, logger, save_path, args, plot=False, zip=False, manual=False):
global label_dict_path
if manual:
save_root = './manul_test_result'
else:
save_root = './result'
if dataset == 'shanghaitech':
label_dict_path = '{}/shanghaitech/GT'.format(args.dataset_path)
elif dataset == 'LAD2000':
label_dict_path = '{}/LAD2000/GT'.format(args.dataset_path)
elif dataset == 'UCF_Crime':
label_dict_path = '{}/UCF_Crime/GT'.format(args.dataset_path)
elif dataset == 'Avenue':
label_dict_path = '{}/Avenue/GT'.format(args.dataset_path)
elif dataset == 'UCSDPed2':
label_dict_path = '{}/UCSDPed2/GT'.format(args.dataset_path)
else:
raise ValueError
with open(file=os.path.join(label_dict_path, 'frame_label.pickle'), mode='rb') as f:
frame_label_dict = pickle.load(f)
with open(file=os.path.join(label_dict_path, 'video_label.pickle'), mode='rb') as f:
video_label_dict = pickle.load(f)
all_predict_np = np.zeros(0)
all_label_np = np.zeros(0)
normal_predict_np = np.zeros(0)
normal_label_np = np.zeros(0)
abnormal_predict_np = np.zeros(0)
abnormal_label_np = np.zeros(0)
for k, v in predict_dict.items():
frame_labels = frame_label_dict[k]
v = cv2.resize(src=v, dsize=(1, frame_labels.shape[0]),interpolation=cv2.INTER_LINEAR).reshape(-1)
predict_dict[k] = v
all_predict_np = np.concatenate((all_predict_np, v))
all_label_np = np.concatenate((all_label_np, frame_labels))
if video_label_dict[k] == [1.]:
abnormal_predict_np = np.concatenate((abnormal_predict_np, v))
abnormal_label_np = np.concatenate((abnormal_label_np, frame_labels))
elif video_label_dict[k] == [0.]:
normal_predict_np = np.concatenate((normal_predict_np, v))
normal_label_np = np.concatenate((normal_label_np, frame_labels))
all_auc_score = roc_auc_score(y_true=all_label_np, y_score=all_predict_np)
binary_all_predict_np = scorebinary(all_predict_np, threshold=0.5)
tn, fp, fn, tp = confusion_matrix(y_true=all_label_np, y_pred=binary_all_predict_np).ravel()
all_ano_false_alarm = fp / (fp + tn)
binary_normal_predict_np = scorebinary(normal_predict_np, threshold=0.5)
# tn, fp, fn, tp = confusion_matrix(y_true=normal_label_np, y_pred=binary_normal_predict_np).ravel()
fp_n = binary_normal_predict_np.sum()
normal_count = normal_label_np.shape[0]
normal_ano_false_alarm = fp_n / normal_count
abnormal_auc_score = roc_auc_score(y_true=abnormal_label_np, y_score=abnormal_predict_np)
binary_abnormal_predict_np = scorebinary(abnormal_predict_np, threshold=0.5)
tn, fp, fn, tp = confusion_matrix(y_true=abnormal_label_np, y_pred=binary_abnormal_predict_np).ravel()
abnormal_ano_false_alarm = fp / (fp + tn)
print('Iteration: {} AUC_score_all_video is {}'.format(itr, all_auc_score))
print('Iteration: {} AUC_score_abnormal_video is {}'.format(itr, abnormal_auc_score))
print('Iteration: {} ano_false_alarm_all_video is {}'.format(itr, all_ano_false_alarm))
print('Iteration: {} ano_false_alarm_normal_video is {}'.format(itr, normal_ano_false_alarm))
print('Iteration: {} ano_false_alarm_abnormal_video is {}'.format(itr, abnormal_ano_false_alarm))
if plot:
anomap(predict_dict, frame_label_dict, save_path, itr, save_root, zip)
if logger:
logger.log_value('Test_AUC_all_video', all_auc_score, itr)
logger.log_value('Test_AUC_abnormal_video', abnormal_auc_score, itr)
logger.log_value('Test_false_alarm_all_video', all_ano_false_alarm, itr)
logger.log_value('Test_false_alarm_normal_video', normal_ano_false_alarm, itr)
logger.log_value('Test_false_alarm_abnormal_video', abnormal_ano_false_alarm, itr)
if os.path.exists(os.path.join(save_root,save_path)) == 0:
os.makedirs(os.path.join(save_root,save_path))
with open(file=os.path.join(save_root, save_path, 'result.txt'), mode='a+') as f:
f.write('itration_{}_AUC_Score_all_video is {}\n'.format(itr, all_auc_score))
f.write('itration_{}_AUC_Score_abnormal_video is {}\n'.format(itr, abnormal_auc_score))
f.write('itration_{}_ano_false_alarm_all_video is {}\n'.format(itr, all_ano_false_alarm))
f.write('itration_{}_ano_false_alarm_normal_video is {}\n'.format(itr, normal_ano_false_alarm))
f.write('itration_{}_ano_false_alarm_abnormal_video is {}\n'.format(itr, abnormal_ano_false_alarm))