-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathget_hyperparameters_dnn_no_outliers.py
111 lines (88 loc) · 4.77 KB
/
get_hyperparameters_dnn_no_outliers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
import pandas as pd
import pickle
from copy import copy
import os
import gc
from adaptive_standardisation import adaptive_standardisation_no_outliers
from epftoolbox.models import hyperparameter_optimizer
import warnings
warnings.filterwarnings(action='ignore')
import random
datasets = ["DE_2023_no_outliers.csv", "SP_2023_no_outliers.csv", "FR_no_outliers.csv", "BE_no_outliers.csv", "NP_no_outliers.csv"]
apply_adaptive_standardisation_list = [False, True]
path_datasets_folder = "Data"
path_hyperparameters_folder = "experimental_files"
path_recalibration_folder = "Results_py"
nlayers = 2
shuffle_train = 1
data_augmentation = 0
new_hyperopt = 1
max_evals = 100
combinations = []
def process_combination(combination):
dataset_name, calibration_window, experiment_id, begin_test_date, end_test_date = combination[0], combination[1], combination[2], combination[3], combination[4]
hyperparameter_optimizer(path_datasets_folder=path_datasets_folder,
path_hyperparameters_folder=path_hyperparameters_folder,
new_hyperopt=new_hyperopt, max_evals=max_evals, nlayers=nlayers, dataset=dataset_name, calibration_window=calibration_window,
shuffle_train=shuffle_train, data_augmentation=0, experiment_id=experiment_id,
begin_test_date=begin_test_date, end_test_date=end_test_date)
for experiment_id in range(1, 5):
for apply_adaptive_standardisation in apply_adaptive_standardisation_list:
for dataset in datasets:
df = pd.read_csv(f"Data//{dataset}")
df['Date'] = pd.to_datetime(df.Date)
if dataset in ["BE_no_outliers.csv", "FR_no_outliers.csv"]:
calibration_window = 4
if apply_adaptive_standardisation:
calibration_window = None
begin_test_date = "04/01/2015 00:00"
end_test_date = "31/12/2016 23:00"
elif dataset in ["DE_2023_no_outliers.csv", "SP_2023_no_outliers.csv"] :
calibration_window = 3
if apply_adaptive_standardisation:
calibration_window = None
begin_test_date = "01/01/2022 00:00"
end_test_date = "31/05/2023 23:00"
elif dataset in ["NP_no_outliers.csv"] :
calibration_window = 4
if apply_adaptive_standardisation:
calibration_window = None
begin_test_date = "27/12/2016 00:00"
end_test_date = "24/12/2018 23:00"
if apply_adaptive_standardisation:
# original_df = copy(df)
df['Simple Date'] = df.Date.dt.strftime("%Y-%m-%d")
df['Hour'] = df.Date.dt.hour
df.columns = ['Date', 'Price', 'Exogenous 1', 'Exogenous 2', 'Price_no_outliers', 'Simple Date', 'Hour']
try:
with open(f'dicts_as_py//dataset_{dataset.replace(".csv", "")}.pkl', 'rb') as f:
dict_new_df = pickle.load(f)
except:
dict_new_df = adaptive_standardisation_no_outliers(df, window_size=7)
with open(f'dicts_as_py//dataset_{dataset.replace(".csv", "")}.pkl', 'wb') as f:
pickle.dump(dict_new_df, f)
df = pd.DataFrame(dict_new_df)[['Date', 'Price', 'Exogenous 1', 'Exogenous 2', 'Price_no_outliers']]
df.to_csv(f"Data//{dataset.replace('.csv', '_as.csv')}", index=False)
df['Date'] = pd.to_datetime(df.Date)
df_scalers = pd.DataFrame({'Date':dict_new_df['Date'], 'scaler':dict_new_df['scaler']})
df = df.set_index('Date')
df.drop('Price', axis=1, inplace=True)
df.columns = ['Exogenous 1', 'Exogenous 2', 'Price']
if apply_adaptive_standardisation:
# original_df = original_df.set_index('Date')
# original_df.columns = ['Price', 'Exogenous 1', 'Exogenous 2']
dataset_name = dataset.replace('.csv', '_as')
else:
dataset_name = dataset.replace('.csv', '')
file_name = 'DNN_hyperparameters_nl' + str(nlayers) + '_dat' + str(dataset_name) + \
'_YT' + str(2) + '_SF' * (shuffle_train) + \
'_DA' * (data_augmentation) + '_CW' + str(calibration_window) + \
'_' + str(experiment_id)
if os.path.isfile("experimental_files//" + file_name):
print(file_name + " already computed")
else:
combinations.append([dataset_name, calibration_window, experiment_id, begin_test_date, end_test_date])
random.shuffle(combinations)
for combination in combinations:
print(combination)
process_combination(combination)