Merge pull request #36 from SiriDB/development

Development

.gitignore

@@ -1,6 +1,7 @@
 test
 test/*
 build
+dist
 .idea
 .idea/*
 .idea/vcs.xml

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.1.0-beta3.2.4] - 2022-05-06
+
+### Changed
+- Removed siridb password from get settings base handler output
+- Refactored manager locks
+- Removed legacy code
+- Moved siridb settings to config file, and `max_in_queue_before_warning` and `min_data_points` to the settings file
+
 ## [0.1.0-beta3.2.3] - 2022-05-02
 
 ### Added

lib/config.py

@@ -20,18 +20,9 @@
         'enable_socket_io_api': 'false',
         'disable_safe_mode': 'false'
     },
-    'events': {
-        'max_in_queue_before_warning': '25'
-    },
     'analyser': {
-        'min_data_points': '100',
         'watcher_interval': '2',
-        'siridb_connection_check_interval': '30',
-        'interval_schedules_series': '3600',
-    }
-}
-
-EMPTY_SETTINGS_FILE = {
+    },
     'siridb': {
         'host': '',
         'port': '',
@@ -48,6 +39,15 @@
     }
 }
 
+EMPTY_SETTINGS_FILE = {
+    'events': {
+        'max_in_queue_before_warning': '25'
+    },
+    'analyser': {
+        'min_data_points': '100'
+    }
+}
+
 
 class EnodoConfigParser(RawConfigParser):
     def __init__(self, env_support=True, **kwargs):
@@ -82,9 +82,7 @@ class Config:
     _settings = None
     min_data_points = None
     watcher_interval = None
-    siridb_connection_check_interval = None
     db = None
-    interval_schedules_series = None
 
     # Siridb
     siridb_host = None
@@ -227,27 +225,21 @@ def write_settings(cls):
 
     @classmethod
     def setup_config_variables(cls):
-        cls.min_data_points = cls.to_int(
-            cls._config.get_r('analyser', 'min_data_points'))
         cls.watcher_interval = cls.to_int(
             cls._config.get_r('analyser', 'watcher_interval'))
-        cls.siridb_connection_check_interval = cls.to_int(
-            cls._config.get_r('analyser', 'siridb_connection_check_interval'))
-        cls.interval_schedules_series = cls.to_int(
-            cls._config.get_r('analyser', 'interval_schedules_series'))
 
         # Enodo
         cls.basic_auth_username = cls._config.get_r(
-           'hub', 'basic_auth_username', required=False, default=None)
+            'hub', 'basic_auth_username', required=False, default=None)
         cls.basic_auth_password = cls._config.get_r(
-           'hub', 'basic_auth_password', required=False, default=None)
+            'hub', 'basic_auth_password', required=False, default=None)
 
         cls.client_max_timeout = cls.to_int(
             cls._config.get_r('hub', 'client_max_timeout'))
         if cls.client_max_timeout < 35:  # min value enforcement
             cls.client_max_timeout = 35
         cls.socket_server_host = cls._config.get_r(
-           'hub', 'internal_socket_server_hostname')
+            'hub', 'internal_socket_server_hostname')
         cls.socket_server_port = cls.to_int(
             cls._config.get_r('hub', 'internal_socket_server_port'))
         cls.save_to_disk_interval = cls.to_int(
@@ -265,7 +257,7 @@ def setup_config_variables(cls):
                 default='false'),
             False)
         cls.base_dir = cls._config.get_r(
-           'hub', 'base_path')
+            'hub', 'base_path')
         cls.disable_safe_mode = cls.to_bool(
             cls._config.get_r('hub', 'disable_safe_mode'), False)
         cls.series_save_path = os.path.join(
@@ -276,38 +268,45 @@ def setup_config_variables(cls):
             cls.base_dir, 'data/outputs.json')
         cls.module_save_path = os.path.join(
             cls.base_dir, 'data/modules.json')
-        cls.max_in_queue_before_warning = cls.to_int(
-            cls._config.get_r('events', 'max_in_queue_before_warning'))
-
-        if not os.path.exists(os.path.join(cls.base_dir, 'data')):
-            os.makedirs(os.path.join(cls.base_dir, 'data'))
 
-    @classmethod
-    def setup_settings_variables(cls):
         # SiriDB
-        cls.siridb_host = cls._settings.get_r('siridb', 'host')
+        cls.siridb_host = cls._config.get_r('siridb', 'host')
         cls.siridb_port = cls.to_int(
-            cls._settings.get_r('siridb', 'port'))
-        cls.siridb_user = cls._settings.get_r('siridb', 'user')
-        cls.siridb_password = cls._settings.get_r('siridb', 'password')
-        cls.siridb_database = cls._settings.get_r('siridb', 'database')
+            cls._config.get_r('siridb', 'port'))
+        cls.siridb_user = cls._config.get_r('siridb', 'user')
+        cls.siridb_password = cls._config.get_r('siridb', 'password')
+        cls.siridb_database = cls._config.get_r('siridb', 'database')
 
         # SiriDB Forecast
-        cls.siridb_output_host = cls._settings.get_r(
+        cls.siridb_output_host = cls._config.get_r(
             'siridb_output',
             'host')
         cls.siridb_output_port = cls.to_int(
-            cls._settings.get_r('siridb_output', 'port'))
-        cls.siridb_output_user = cls._settings.get_r(
+            cls._config.get_r('siridb_output', 'port'))
+        cls.siridb_output_user = cls._config.get_r(
             'siridb_output',
             'user')
-        cls.siridb_output_password = cls._settings.get_r(
+        cls.siridb_output_password = cls._config.get_r(
             'siridb_output',
             'password')
-        cls.siridb_output_database = cls._settings.get_r(
+        cls.siridb_output_database = cls._config.get_r(
             'siridb_output',
             'database')
 
+        if not os.path.exists(os.path.join(cls.base_dir, 'data')):
+            os.makedirs(os.path.join(cls.base_dir, 'data'))
+
+    @classmethod
+    def setup_settings_variables(cls):
+        # TODO set default in one place/overview
+        cls.max_in_queue_before_warning = cls.to_int(cls._settings.get_r(
+            'events', 'max_in_queue_before_warning',
+            required=False, default=25))
+        cls.min_data_points = cls.to_int(
+            cls._settings.get_r(
+                'analyser', 'min_data_points', required=False,
+                default=100))
+
     @staticmethod
     def to_int(val):
         return_val = None
@@ -328,26 +327,32 @@ def to_bool(v, default):
         else:
             return default
 
+    @staticmethod
+    def _remove_dict_key_recursive(data, keys):
+        Config._remove_dict_key_recursive(
+            data[keys[0]],
+            keys[1:]) if len(keys) > 1 else data.pop(
+            keys[0],
+            None)
+
     @classmethod
-    def get_settings(cls):
-        return cls._settings._sections
+    def get_settings(cls, include_secrets=True):
+        if not include_secrets:
+            secret_paths = []
+            data = cls._settings._sections
+        for secret in secret_paths:
+            cls._remove_dict_key_recursive(data, secret)
+
+        return data
 
     @staticmethod
     def is_runtime_configurable(section, key):
         _is_runtime_configurable = {
-            "siridb": [
-                "host",
-                "port",
-                "user",
-                "password",
-                "database"
+            "events": [
+                "max_in_queue_before_warning"
             ],
-            "siridb_output": [
-                "host",
-                "port",
-                "user",
-                "password",
-                "database"
+            "analyser": [
+                "min_data_points"
             ]
         }
 

lib/events/enodoeventmanager.py → lib/eventmanager.py

@@ -12,7 +12,7 @@
 from lib.socketio import SUBSCRIPTION_CHANGE_TYPE_ADD, \
     SUBSCRIPTION_CHANGE_TYPE_UPDATE, SUBSCRIPTION_CHANGE_TYPE_DELETE
 from lib.serverstate import ServerState
-from lib.util import save_disk_data, load_disk_data
+from lib.util import save_disk_data, load_disk_data, cls_lock
 
 ENODO_EVENT_ANOMALY_DETECTED = "event_anomaly_detected"
 ENODO_EVENT_JOB_QUEUE_TOO_LONG = "job_queue_too_long"
@@ -206,32 +206,22 @@ class EnodoEventManager:
     outputs = None
     # Next id is always current. will be incremented when setting new id
     _next_output_id = None
-    _locked = False
+    _lock = None
     _max_output_id = None
 
     @classmethod
     async def async_setup(cls):
         cls.outputs = []
         cls._next_output_id = 0
         cls._max_output_id = 1000
+        cls._lock = asyncio.Lock()
 
     @classmethod
-    async def _lock(cls):
-        while cls._locked is True:
-            await asyncio.sleep(0.1)
-        cls._locked = True
-
-    @classmethod
-    async def _unlock(cls):
-        cls._locked = False
-
-    @classmethod
+    @cls_lock()
     async def _get_next_output_id(cls):
-        await cls._lock()
         if cls._next_output_id + 1 >= cls._max_output_id:
             cls._next_output_id = 0
         cls._next_output_id += 1
-        await cls._unlock()
         return cls._next_output_id
 
     @classmethod
@@ -261,20 +251,18 @@ async def remove_event_output(cls, output_id):
         return False
 
     @classmethod
+    @cls_lock()
     async def _remove_event_output(cls, output):
-        await cls._lock()
         cls.outputs.remove(output)
         await internal_updates_event_output_subscribers(
             SUBSCRIPTION_CHANGE_TYPE_DELETE, output.rid)
-        await cls._unlock()
 
     @classmethod
+    @cls_lock()
     async def _update_event_output(cls, output, data):
-        await cls._lock()
         output.update(data)
         await internal_updates_event_output_subscribers(
             SUBSCRIPTION_CHANGE_TYPE_UPDATE, output.to_dict())
-        await cls._unlock()
 
     @classmethod
     async def handle_event(cls, event, series=None):