-
Notifications
You must be signed in to change notification settings - Fork 1
/
overfitting_study.py
72 lines (60 loc) · 3.07 KB
/
overfitting_study.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
__author__ = "shekkizh"
import os
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
import seaborn as sbn
import json
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
sbn.set(font_scale=1.5)
def k_param_study(model_path, knn_values, data_type="train"):
params = json.load(open(os.path.join(model_path, "parameters.json")))
n_epochs = params["epochs"]
plt.figure(1, figsize=(10, 6))
plt.gca().xaxis.set_major_locator(MaxNLocator(integer=True))
cmap = plt.get_cmap("tab10")
epoch_range = range(1, n_epochs + 1)
for i, k in enumerate(knn_values):
calibrate_results_path = os.path.join(model_path, '%s_calibrate_results_%d' % (data_type, k))
graph_name = "NNK"
results = np.load(os.path.join(calibrate_results_path, 'nnk_calibrate_data.npz'))
nnk_classification_error_rate = results['nnk_classification_error_rate']
plt.plot(epoch_range, nnk_classification_error_rate, color=cmap(i * 2 + 1), marker='x',
linestyle='-', linewidth=1.5, label='%s (k=%d)' % (graph_name, k))
# #########################################################################################
graph_name = "KNN"
results = np.load(os.path.join(calibrate_results_path, 'knn_calibrate_data.npz'))
knn_classification_error_rate = results['knn_classification_error_rate']
plt.plot(epoch_range, knn_classification_error_rate, color=cmap(i * 2 + 2), marker='o',
linestyle=':', linewidth=1.5, label='%s (k=%d)' % (graph_name, k))
# #########################################################################################
model_error_rate = results['model_error_rate']
results = np.load(os.path.join(model_path, 'SVC_calibrate_results', 'SVC_calibrate_data_CV_5.npz'))
cv_error = 1 - results['svm_classification_%s_error_rate' % data_type]
plt.plot(range(1, n_epochs + 1), np.mean(cv_error, axis=1), color=cmap(len(knn_values) * 2 + 1), marker='s',
linestyle='-.', linewidth=1.5, label='%s (%d-Fold CV)' % ('SVM', 5))
plt.plot(range(1, n_epochs + 1), model_error_rate, color='k', marker='s',
linestyle='-', linewidth=1.5, label='Model')
fig = plt.gcf()
fig.legend(loc=9, ncol=8, fontsize='small', handletextpad=0.1, labelspacing=0.1)
plt.grid(linestyle='dashed')
plt.xlabel("epochs")
plt.xlim([0.5, 20.5])
plt.ylim([0.1, 0.7])
plt.ylabel('Avg. classifier error')
plt.savefig(os.path.join(model_path, '%s_classifier_error_rate.eps' % data_type),
bbox_inches='tight')
# plt.show()
plt.close()
if __name__ == "__main__":
model_paths = [
"logs/conv2d_models_cifar10_layer_size_16_regularized_True", # Underparam
"logs/conv2d_models_cifar10_layer_size_32_regularized_True", # Regularized
"logs/conv2d_models_cifar10_layer_size_32_regularized_False" # Overfit
]
knn_values = [25, 50, 75]
data_type = "train"
for model_path in model_paths:
k_param_study(model_path, knn_values, "train")