Merge pull request #60 from redBorder/add_direct_data_execution

PR-56: Add direct data execution

.github/workflows/lint.yml

@@ -17,13 +17,13 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.8 and pip
+    - name: Set up Python 3.9 and pip
       run: |
         sudo add-apt-repository ppa:deadsnakes/ppa -y
         sudo apt update
-        sudo apt install -y python3.8 python3.8-distutils
+        sudo apt install -y python3.9 python3.9-distutils
         curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-        sudo python3.8 get-pip.py
+        sudo python3.9 get-pip.py
 
     - name: Install pylint
       run: |

.github/workflows/security.yml

@@ -17,13 +17,13 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.8 and pip
+    - name: Set up Python 3.9 and pip
       run: |
         sudo add-apt-repository ppa:deadsnakes/ppa -y
         sudo apt update
-        sudo apt install -y python3.8 python3.8-distutils
+        sudo apt install -y python3.9 python3.9-distutils
         curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-        sudo python3.8 get-pip.py
+        sudo python3.9 get-pip.py
 
     - name: Install safety for security checks
       run: |

.github/workflows/tests.yml

@@ -17,13 +17,13 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.8 and pip
+    - name: Set up Python 3.9 and pip
       run: |
         sudo add-apt-repository ppa:deadsnakes/ppa -y
         sudo apt update
-        sudo apt install -y python3.8 python3.8-distutils
+        sudo apt install -y python3.9 python3.9-distutils
         curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-        sudo python3.8 get-pip.py
+        sudo python3.9 get-pip.py
 
     - name: Install project dependencies
       run: |
@@ -51,21 +51,21 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.8 and pip
+    - name: Set up Python 3.9 and pip
       run: |
         sudo add-apt-repository ppa:deadsnakes/ppa -y
         sudo apt update
-        sudo apt install -y python3.8 python3.8-distutils
+        sudo apt install -y python3.9 python3.9-distutils
         curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-        sudo python3.8 get-pip.py
+        sudo python3.9 get-pip.py
 
     - name: Install dependencies
       run: |
         pip install -r resources/src/requirements.txt
 
     - name: Run outliers
       run: |
-        ENVIRONMENT=test python3.8 resources/src/__main__.py
+        ENVIRONMENT=test python3.9 resources/src/__main__.py
       
 
         

README.md

@@ -15,6 +15,14 @@ Main package to install redborder AI outliers in Rocky Linux 9
 
 - Rocky Linux 9
 
+## Running the example
+This code shows runs the outlier detection on a mock dataset. Its recomended to use pipenv or similar to avoid overwritting dependencies.
+```bash
+git clone [email protected]:redBorder/rb-aioutliers.git
+cd rb-aioutliers
+pip install -r resources/src/requirements.txt
+bash resources/src/example/run_example.sh
+```
 ## Installation
 
 1. yum install epel-release && rpm -ivh http://repo.redborder.com/redborder-repo-0.0.3-1.el7.rb.noarch.rpm
@@ -212,4 +220,4 @@ query=base64_string
 - Miguel Álvarez Adsuara <[email protected]>
 - Pablo Rodriguez Flores <[email protected]>
 
-LICENSE: GENERAL PUBLIC LICENSE, Version 2, June 1991
+LICENSE: GENERAL PUBLIC LICENSE, Version 2, June 1991

resources/src/ai/outliers.py

@@ -77,8 +77,6 @@ def __init__(self, model_file, model_config_file):
             self.std_loss = float(general_section.get('STD_LOSS', 1.0))
             self.window_size = int(general_section.get('WINDOW_SIZE', 1))
             self.num_window = int(general_section.get('NUM_WINDOWS', 1))
-            self.loss_mult_metric = float(general_section.get('LOSS_MULT_METRIC', 1.0))
-            self.loss_mult_minute = float(general_section.get('LOSS_MULT_MINUTE', 1.0))
         except Exception as e:
             logger.logger.error(f"Could not load model conif: {e}")
             raise e
@@ -150,13 +148,8 @@ def descale(self, data):
 
     def model_loss(self, y_true, y_pred, single_value=True):
         """
-        Calculate the weighted loss for the model.
-        Compares the input with boolean-valued tensors IS_METRIC and IS_MINUTE.
-        Where IS_METRIC is true, the value of the input is multiplied by mult1,
-        where IS_MINUTE is true, the value of the input is multiplied by mult2,
-        otherwise, the value is left unchanged.
-        Then, the difference between both tensors is evaluated and a log_cosh loss
-        is applied.
+        Calculate the loss of the model as a mean absolute error. May be computed fo each separate
+        element of the input tensor or for the whole tensor.
 
         Args:
             y_true (tf.Tensor): True target values.
@@ -168,16 +161,7 @@ def model_loss(self, y_true, y_pred, single_value=True):
         """
         y_true = tf.cast(y_true, tf.bfloat16)
         y_pred = tf.cast(y_pred, tf.bfloat16)
-        num_metrics = len(self.metrics)
-        num_features = len(self.columns)
-        is_metric = (tf.range(num_features) < num_metrics)
-        is_minute = (tf.range(num_features) == num_metrics)
-        mult_true = tf.where(is_metric, self.loss_mult_metric * y_true, y_true)
-        mult_true = tf.where(is_minute, self.loss_mult_minute * mult_true, mult_true)
-        mult_pred = tf.where(is_metric, self.loss_mult_metric * y_pred, y_pred)
-        mult_pred = tf.where(is_minute, self.loss_mult_minute * mult_pred, mult_pred)
-        loss = tf.math.abs(mult_true-mult_pred)
-        loss = loss-tf.math.log(tf.cast(2.0, tf.bfloat16))+tf.math.log1p(tf.math.exp(-2.0*loss))
+        loss = tf.math.abs(y_true-y_pred)
         if single_value:
             loss = tf.reduce_mean(loss)
         return loss

resources/src/ai/traffic.ini

@@ -3,10 +3,8 @@ metrics = bits_per_sec_per_client, bps, bytes, bytes_per_client, clients, flows,
 timestamp = minute, weekday_0, weekday_1, weekday_2, weekday_3, weekday_4, weekday_5, weekday_6
 
 [General]
-avg_loss = 0.09741491896919627
-std_loss = 0.11098885675977664
+avg_loss = 0.004971476081581194
+std_loss = 0.007315154341724864
 window_size = 16
 num_windows = 2
-loss_mult_metric = 20.0
-loss_mult_minute = 10.0
 

resources/src/ai/trainer.py

@@ -92,8 +92,6 @@ def save_model(self, save_model_file, save_config_file):
         general_section['STD_LOSS'] = str(self.std_loss)
         general_section['WINDOW_SIZE'] = str(self.window_size)
         general_section['NUM_WINDOWS'] = str(self.num_window)
-        general_section['LOSS_MULT_METRIC'] = str(self.loss_mult_metric)
-        general_section['LOSS_MULT_MINUTE'] = str(self.loss_mult_minute)
         with open(save_config_file, 'w') as configfile:
             new_model_config.write(configfile)