implement global distinction of device types

algo/curve_anomaly/cont_device.py

@@ -165,7 +165,7 @@ def test(data_list, model, use_cuda, anomalies, model_input_window_length=205):
     reconstruction_dtw_errors = get_dtw_errors(model_input_data_array, model, use_cuda)
     #anomalous_reconstruction_area_errors = error_calculation.get_anomalous_indices(reconstruction_area_errors,0.05)
     #anomalous_reconstruction_pcm_errors = error_calculation.get_anomalous_indices(reconstruction_pcm_errors,0.1)
-    anomalous_reconstruction_dtw_errors = error_calculation.get_anomalous_indices(reconstruction_dtw_errors,0.01)
+    anomalous_reconstruction_dtw_errors = error_calculation.get_anomalous_indices(reconstruction_dtw_errors,0.005)
     #anomalous_reconstruction_indices = error_calculation.get_anomalous_indices(reconstruction_pcm_errors,0.03)+error_calculation.get_anomalous_indices(reconstruction_dtw_errors,0.05)+error_calculation.get_anomalous_indices(reconstruction_area_errors,0.01)
     '''if model_input_data_array.shape[0]-1 in anomalous_reconstruction_area_errors:
         anomalous_time_window = data_series[-model_input_window_length:]

algo/curve_anomaly/curve_utils.py

@@ -9,7 +9,7 @@ def batch_train(data_list, first_data_time, last_training_time, device_type, mod
         current_timestamp = utils.todatetime(data_list[-1][0]).tz_localize(None)
         if current_timestamp-last_training_time.tz_localize(None) >= pd.Timedelta(14, 'days'): 
             if device_type == 'cont_device':
-                if last_training_time == first_data_time:
+                if last_training_time.tz_localize(None) == first_data_time.tz_localize(None):
                     model = cont_device.Autoencoder(32)
                     if use_cuda:
                         model = model.cuda()
@@ -22,7 +22,7 @@ def batch_train(data_list, first_data_time, last_training_time, device_type, mod
             return last_training_time, model, training_performance
 
 def test(data_list, first_data_time, last_training_time, device_type, model, use_cuda, anomalies, loads, init_median):
-        if device_type == 'cont_device' and last_training_time > first_data_time:
+        if device_type == 'cont_device' and last_training_time.tz_localize(None) > first_data_time.tz_localize(None):
             output, anomalies = cont_device.test(data_list, model, use_cuda, anomalies)
             return output, loads, anomalies
         elif device_type == 'load_device':

algo/curve_anomaly/load_device.py

@@ -9,7 +9,6 @@ def extract_loads(time_series, init_median):
     list_of_loads = []
     list_of_load_inds = []
     new_load = []
-    end_check = []
     active = False
     for i in range(len(time_series)):
         if active == True:
@@ -21,9 +20,6 @@ def extract_loads(time_series, init_median):
             elif time_series[i] <= init_median + 1 and start_of_end:
                 if time_series.index[i] - start_of_end >= pd.Timedelta(10,"min"): # If values where constantly below the threshold for 10min, the load has stopped.
                     active = False
-                    print(new_load)
-                    import time
-                    time.sleep(5)
                     list_of_load_inds.append(new_load)
                     new_load = []
         elif active == False:    
@@ -69,7 +65,7 @@ def train_test(data_list, loads, anomalies, init_median):
     if len(loads) > old_number_of_loads:
         list_of_normalized_loads = [preprocessing.normalize_data(load) for load in loads]
         anomalous_length_indices = find_anomalous_lengths(list_of_normalized_loads)
-        if len(loads)-1 in anomalous_length_indices:
+        if len(loads)-1 in anomalous_length_indices and len(loads) >= 15:
             anomalies.append((loads[-1],'length of load'))
             util.logger.debug('A load of anomalous length just ended!')
             return 'load_device_anomaly_length', loads, anomalies

algo/detector.py

@@ -59,6 +59,15 @@ def __init__(
             consumption_explorer = consumption_anomaly.Consumption_Explorer(os.path.join(data_path, "consumption_explorer"))
 
 
+    def update_device_type(self, device_type):
+        for detector in self.active_detectors:
+            detector.device_type = device_type
+
+    def update_init_median(self, init_median):
+        for detector in self.active_detectors:
+            detector.init_median = init_median
+
+
     def check_input(self, value, timestamp):
         anomaly_results = []
         for detector in self.active_detectors:

algo/operator.py

@@ -148,8 +148,9 @@ def run(self, data, selector='energy_func', device_id=''):
         device_detector = self.get_device_detectors(input_id)
         anomalies_found = None
         timestamp_without_tz = timestamp.tz_localize(None)
-        #if self.input_is_real_time(timestamp) and not operator_is_init:
-        device_detector.start_freq_loop()
+        if self.input_is_real_time(timestamp):
+            device_detector.start_freq_loop()
+
         util.logger.debug(f"{LOG_PREFIX}: Check input for anomalies")
         anomalies_found = device_detector.check_input(value, timestamp_without_tz)
         util.logger.debug(f"{LOG_PREFIX}: Found Anomalies: {anomalies_found}")
@@ -175,8 +176,10 @@ def run(self, data, selector='energy_func', device_id=''):
 
         if not self.device_type:
             self.device_type, self.init_median = self.get_device_type()
+            self.device_detectors[input_id].update_device_type(self.device_type)
+            self.device_detectors[input_id].update_init_median(self.init_median)
 
-        if anomalies_found:
+        if anomalies_found and not operator_is_init:
             return anomalies_found
 
     def stop(self):

algo/point_outlier/point_explorer.py

@@ -9,7 +9,7 @@
 class Point_Explorer(utils.StdPointOutlierDetector):
     def __init__(self, data_path, device_type, init_median):
         super().__init__(data_path)
-        self.active = False # Introduce this variable to constantly check
+        self.active = False # Introduce this variable to constantly check if device is turned on or off.
         self.device_type = device_type
         self.init_median = init_median
 
@@ -44,15 +44,15 @@ def update_with_new_value(self, value, timestamp):
         if self.device_type == "load_device":
             if not self.active and value > self.init_median + 10:
                 self.active = True
-                start_of_end = None
+                self.start_of_end = None
             elif self.active:
-                if value <= self.init_median + 1 and not start_of_end:
-                    start_of_end = timestamp
-                elif value <= self.init_median + 1 and start_of_end:
-                    if timestamp - start_of_end >= pd.Timedelta(10,"min"):
+                if value <= self.init_median + 1 and not self.start_of_end:
+                    self.start_of_end = timestamp
+                elif value <= self.init_median + 1 and self.start_of_end:
+                    if timestamp - self.start_of_end >= pd.Timedelta(10,"min"):
                         self.active = False
-                elif value > self.init_median + 1 and start_of_end:
-                    start_of_end = None
+                elif value > self.init_median + 1 and self.start_of_end:
+                    self.start_of_end = None
 
         if self.active or self.device_type == "cont_device":
             self.update(value)

algo/utils.py

@@ -43,7 +43,6 @@ def save(self):
         current_stddev_path = self.filename_dict["current_stddev"]
         current_mean_path = self.filename_dict["current_mean"]
         num_datepoints_path = self.filename_dict["num_datepoints"]
-        first_data_time_path = self.filename_dict["first_data_time"]
 
         with open(current_stddev_path, 'wb') as f:
             pickle.dump(self.current_stddev, f)
@@ -76,18 +75,18 @@ def load_data(self, current_stddev, current_mean, num_datepoints):
     def point_is_anomalous_high(self, point):
         if self.num_datepoints < 2:
             return False
-        return point > self.current_mean + 5*self.current_stddev
+        return point > self.get_upper_threshold()
 
     def point_is_anomalous_low(self, point):
         if self.num_datepoints < 2:
             return False
-        return point < self.current_mean - 5*self.current_stddev
+        return point < self.get_lower_threshold()
 
     def get_upper_threshold(self):
-        return self.current_mean + 5*self.current_stddev
+        return self.current_mean + 30*self.current_stddev
 
     def get_lower_threshold(self):
-        return self.current_mean + 5*self.current_stddev
+        return self.current_mean - 30*self.current_stddev
 
     def update(self, point):
         self.current_stddev = self.calculate_std(point, self.current_stddev, self.current_mean, self.num_datepoints)