Merge pull request #37 from SiriDB/development

unknown module support

CHANGELOG.md

@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [unreleased] - yyyy-mm-dd
+
+## [0.1.0-beta3.2.5] - 2022-05-09
+
+### Added
+- Support for adding series with unknown modules
+
+### Changed
+- Known modules in memory only, not saved to disk
+- SiriDB config renamed to `siridb_data`
+
 ## [0.1.0-beta3.2.4] - 2022-05-06
 
 ### Changed

lib/config.py

@@ -23,7 +23,7 @@
     'analyser': {
         'watcher_interval': '2',
     },
-    'siridb': {
+    'siridb_data': {
         'host': '',
         'port': '',
         'user': '',
@@ -270,12 +270,12 @@ def setup_config_variables(cls):
             cls.base_dir, 'data/modules.json')
 
         # SiriDB
-        cls.siridb_host = cls._config.get_r('siridb', 'host')
+        cls.siridb_host = cls._config.get_r('siridb_data', 'host')
         cls.siridb_port = cls.to_int(
-            cls._config.get_r('siridb', 'port'))
-        cls.siridb_user = cls._config.get_r('siridb', 'user')
-        cls.siridb_password = cls._config.get_r('siridb', 'password')
-        cls.siridb_database = cls._config.get_r('siridb', 'database')
+            cls._config.get_r('siridb_data', 'port'))
+        cls.siridb_user = cls._config.get_r('siridb_data', 'user')
+        cls.siridb_password = cls._config.get_r('siridb_data', 'password')
+        cls.siridb_database = cls._config.get_r('siridb_data', 'database')
 
         # SiriDB Forecast
         cls.siridb_output_host = cls._config.get_r(

lib/jobmanager.py

@@ -142,6 +142,9 @@ async def clear_jobs(cls):
     async def create_job(cls, job_config_name, series_name):
         series = await SeriesManager.get_series(series_name)
         await series.set_job_status(job_config_name, JOB_STATUS_OPEN)
+        series.state.set_job_check_status(
+            job_config_name,
+            "Job created")
         job_config = series.get_job(job_config_name)
         job_id = await cls._get_next_job_id()
         job = EnodoJob(job_id, series_name, job_config,

lib/modulemanager.py

@@ -1,10 +1,5 @@
-import os
-import logging
-
 from enodo import EnodoModule
 
-from lib.config import Config
-from lib.util import load_disk_data, save_disk_data
 from lib.socketio import SUBSCRIPTION_CHANGE_TYPE_ADD, \
     SUBSCRIPTION_CHANGE_TYPE_DELETE
 
@@ -46,33 +41,3 @@ async def remove_module(cls, name):
         module = await cls.get_module(name)
         cls.modules.remove(module)
         await cls._update_cb(SUBSCRIPTION_CHANGE_TYPE_DELETE, name)
-
-    @classmethod
-    async def load_from_disk(cls):
-        try:
-            if not os.path.exists(Config.module_save_path):
-                raise Exception()
-            data = load_disk_data(Config.module_save_path)
-        except Exception as e:
-            data = {}
-
-        if isinstance(data, list):
-            for module_data in data:
-                module = EnodoModule(**module_data)
-                cls.modules.append(module)
-
-    @classmethod
-    async def save_to_disk(cls):
-        module_list = []
-        if cls.modules is None:
-            return
-        for module in cls.modules:
-            module_list.append(module)
-
-        try:
-            save_disk_data(Config.module_save_path, module_list)
-        except Exception as e:
-            logging.error("Something went wrong when writing"
-                          "enodo modules to disk")
-            logging.debug(f"Corresponding error: {e}, "
-                          f'exception class: {e.__class__.__name__}')

lib/series/series.py

@@ -4,6 +4,7 @@
 from enodo.jobs import JOB_TYPE_BASE_SERIES_ANALYSIS, JOB_STATUS_NONE, \
     JOB_STATUS_DONE
 from enodo.model.config.series import SeriesConfigModel, SeriesState
+from lib.modulemanager import EnodoModuleManager
 
 
 class Series:
@@ -111,6 +112,12 @@ async def is_job_due(self, job_config_name):
         if job_status in [JOB_STATUS_NONE, JOB_STATUS_DONE]:
             job_config = self.series_config.get_config_for_job(
                 job_config_name)
+            module = await EnodoModuleManager.get_module(job_config.module)
+            if module is None:
+                self.state.set_job_check_status(
+                    job_config_name,
+                    "Unknown module")
+                return False
             if job_config.requires_job is not None:
                 required_job_status = self.state.get_job_status(
                     job_config.requires_job)

lib/webserver/basehandler.py

@@ -191,18 +191,17 @@ async def resp_add_series(cls, data):
             module_parameters = job_config.module_params
 
             module = await EnodoModuleManager.get_module(job_config.module)
-            if module is None:
-                return {'error': 'Unknown module'}, 400
-            if module_parameters is None and len(
-                    module.module_arguments.keys()) > 0:
-                return {'error': 'Missing module parameters'}, 400
-            for m_args in module.module_arguments:
-                if job_config.job_type in m_args.get("job_types", []) and \
-                        m_args.get("required") and \
-                        m_args.get('name') not in module_parameters.keys():
-                    return {'error': "Missing required module parameter '"
-                            f"{m_args.get('name')}' for job type "
-                            f"{job_config.job_type}"}, 400
+            if module is not None:
+                if module_parameters is None and len(
+                        module.module_arguments.keys()) > 0:
+                    return {'error': 'Missing module parameters'}, 400
+                for m_args in module.module_arguments:
+                    if job_config.job_type in m_args.get("job_types", []) and \
+                            m_args.get("required") and \
+                            m_args.get('name') not in module_parameters.keys():
+                        return {'error': "Missing required module parameter '"
+                                f"{m_args.get('name')}' for job type "
+                                f"{job_config.job_type}"}, 400
 
         if not await SeriesManager.add_series(data):
             return {'error': 'Something went wrong when adding the series. \
@@ -221,24 +220,21 @@ async def resp_update_series(cls, series_name, data):
         Returns:
             dict: dict with data
         """
-        required_fields = ['config']
-        if not all(required_field in data
-                   for required_field in required_fields):
-            return {'error': 'Something went wrong when updating the series. \
-                Missing required fields'}, 400
-        series_config = SeriesConfigModel(**data.get('config'))
+        try:
+            series_config = SeriesConfigModel(**data.get('config'))
+        except Exception as e:
+            return {'error': 'Invalid series config', 'message': str(e)}, 400
         for job_config in list(series_config.job_config.values()):
             module_parameters = job_config.module_params
 
             module = await EnodoModuleManager.get_module(job_config.module)
-            if module is None:
-                return {'error': 'Unknown module'}, 400
-            if module_parameters is None and len(
-                    module.module_arguments.keys()) > 0:
-                return {'error': 'Missing required fields'}, 400
-            for key in module.module_arguments:
-                if key not in module_parameters.keys():
-                    return {'error': f'Missing required field {key}'}, 400
+            if module is not None:
+                if module_parameters is None and len(
+                        module.module_arguments.keys()) > 0:
+                    return {'error': 'Missing required fields'}, 400
+                for key in module.module_arguments:
+                    if key not in module_parameters.keys():
+                        return {'error': f'Missing required field {key}'}, 400
 
         series = await SeriesManager.get_series(series_name)
         if series is None:

requirements.txt

@@ -2,9 +2,8 @@ aiohttp==3.8.1
 aiojobs==1.0.0
 aiohttp_basicauth==1.0.0
 aiohttp_cors==0.7.0
-jinja2==3.1.0
-psutil==5.9.0
-python-enodo==0.2.15
+jinja2==3.1.2
+python-enodo==0.2.18
 python-socketio==5.5.2
 qpack==0.0.19
 siridb-connector==2.0.8

server.py

@@ -107,7 +107,6 @@ async def start_up(self):
         await EnodoEventManager.load_from_disk()
         await EnodoModuleManager.async_setup(
             SocketIoHandler.internal_updates_enodo_modules_subscribers)
-        await EnodoModuleManager.load_from_disk()
 
         scheduler = ServerState.scheduler
         self._watch_series_task = await scheduler.spawn(self.watch_series())
@@ -202,6 +201,13 @@ async def _check_for_jobs(self, series, series_name):
         # Check if base analysis has already run
         if series.base_analysis_status() == JOB_STATUS_NONE:
             base_analysis_job = series.base_analysis_job
+            module = await EnodoModuleManager.get_module(
+                base_analysis_job.module)
+            if module is None:
+                series.state.set_job_check_status(
+                    base_analysis_job.config_name,
+                    "Unknown module")
+                return
             await EnodoJobManager.create_job(
                 base_analysis_job.config_name, series_name)
         # Only continue if base analysis has finished
@@ -249,8 +255,6 @@ async def watch_series(self):
                 except Exception as e:
                     logging.error(
                         f"Something went wrong when trying to create new job")
-                    import traceback
-                    traceback.print_exc()
                     logging.debug(
                         f"Corresponding error: {e}, "
                         f'exception class: {e.__class__.__name__}')
@@ -264,7 +268,6 @@ async def _save_to_disk():
         await SeriesManager.save_to_disk()
         await EnodoJobManager.save_to_disk()
         await EnodoEventManager.save_to_disk()
-        await EnodoModuleManager.save_to_disk()
 
     async def save_to_disk(self):
         """Save configs to disk on a set interval