From 10a16e424642ae6e3ad8cb17747b80619cce3cf4 Mon Sep 17 00:00:00 2001 From: timoj Date: Mon, 9 May 2022 15:39:45 +0200 Subject: [PATCH] unknown module support --- CHANGELOG.md | 11 +++++++++ lib/config.py | 12 ++++----- lib/jobmanager.py | 3 +++ lib/modulemanager.py | 35 -------------------------- lib/series/series.py | 7 ++++++ lib/webserver/basehandler.py | 48 +++++++++++++++++------------------- requirements.txt | 5 ++-- server.py | 11 ++++++--- 8 files changed, 58 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 438ac07..6ae46d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [unreleased] - yyyy-mm-dd + +## [0.1.0-beta3.2.5] - 2022-05-09 + +### Added +- Support for adding series with unknown modules + +### Changed +- Known modules in memory only, not saved to disk +- SiriDB config renamed to `siridb_data` + ## [0.1.0-beta3.2.4] - 2022-05-06 ### Changed diff --git a/lib/config.py b/lib/config.py index 790392f..bdd30df 100644 --- a/lib/config.py +++ b/lib/config.py @@ -23,7 +23,7 @@ 'analyser': { 'watcher_interval': '2', }, - 'siridb': { + 'siridb_data': { 'host': '', 'port': '', 'user': '', @@ -270,12 +270,12 @@ def setup_config_variables(cls): cls.base_dir, 'data/modules.json') # SiriDB - cls.siridb_host = cls._config.get_r('siridb', 'host') + cls.siridb_host = cls._config.get_r('siridb_data', 'host') cls.siridb_port = cls.to_int( - cls._config.get_r('siridb', 'port')) - cls.siridb_user = cls._config.get_r('siridb', 'user') - cls.siridb_password = cls._config.get_r('siridb', 'password') - cls.siridb_database = cls._config.get_r('siridb', 'database') + cls._config.get_r('siridb_data', 'port')) + cls.siridb_user = cls._config.get_r('siridb_data', 'user') + cls.siridb_password = cls._config.get_r('siridb_data', 'password') + cls.siridb_database = cls._config.get_r('siridb_data', 'database') # SiriDB Forecast cls.siridb_output_host = cls._config.get_r( diff --git a/lib/jobmanager.py b/lib/jobmanager.py index 4086b5d..55f0873 100644 --- a/lib/jobmanager.py +++ b/lib/jobmanager.py @@ -142,6 +142,9 @@ async def clear_jobs(cls): async def create_job(cls, job_config_name, series_name): series = await SeriesManager.get_series(series_name) await series.set_job_status(job_config_name, JOB_STATUS_OPEN) + series.state.set_job_check_status( + job_config_name, + "Job created") job_config = series.get_job(job_config_name) job_id = await cls._get_next_job_id() job = EnodoJob(job_id, series_name, job_config, diff --git a/lib/modulemanager.py b/lib/modulemanager.py index f78cdb5..ef846a5 100644 --- a/lib/modulemanager.py +++ b/lib/modulemanager.py @@ -1,10 +1,5 @@ -import os -import logging - from enodo import EnodoModule -from lib.config import Config -from lib.util import load_disk_data, save_disk_data from lib.socketio import SUBSCRIPTION_CHANGE_TYPE_ADD, \ SUBSCRIPTION_CHANGE_TYPE_DELETE @@ -46,33 +41,3 @@ async def remove_module(cls, name): module = await cls.get_module(name) cls.modules.remove(module) await cls._update_cb(SUBSCRIPTION_CHANGE_TYPE_DELETE, name) - - @classmethod - async def load_from_disk(cls): - try: - if not os.path.exists(Config.module_save_path): - raise Exception() - data = load_disk_data(Config.module_save_path) - except Exception as e: - data = {} - - if isinstance(data, list): - for module_data in data: - module = EnodoModule(**module_data) - cls.modules.append(module) - - @classmethod - async def save_to_disk(cls): - module_list = [] - if cls.modules is None: - return - for module in cls.modules: - module_list.append(module) - - try: - save_disk_data(Config.module_save_path, module_list) - except Exception as e: - logging.error("Something went wrong when writing" - "enodo modules to disk") - logging.debug(f"Corresponding error: {e}, " - f'exception class: {e.__class__.__name__}') diff --git a/lib/series/series.py b/lib/series/series.py index 833bc9b..85bd2f3 100644 --- a/lib/series/series.py +++ b/lib/series/series.py @@ -4,6 +4,7 @@ from enodo.jobs import JOB_TYPE_BASE_SERIES_ANALYSIS, JOB_STATUS_NONE, \ JOB_STATUS_DONE from enodo.model.config.series import SeriesConfigModel, SeriesState +from lib.modulemanager import EnodoModuleManager class Series: @@ -111,6 +112,12 @@ async def is_job_due(self, job_config_name): if job_status in [JOB_STATUS_NONE, JOB_STATUS_DONE]: job_config = self.series_config.get_config_for_job( job_config_name) + module = await EnodoModuleManager.get_module(job_config.module) + if module is None: + self.state.set_job_check_status( + job_config_name, + "Unknown module") + return False if job_config.requires_job is not None: required_job_status = self.state.get_job_status( job_config.requires_job) diff --git a/lib/webserver/basehandler.py b/lib/webserver/basehandler.py index 06b95fd..851a415 100644 --- a/lib/webserver/basehandler.py +++ b/lib/webserver/basehandler.py @@ -191,18 +191,17 @@ async def resp_add_series(cls, data): module_parameters = job_config.module_params module = await EnodoModuleManager.get_module(job_config.module) - if module is None: - return {'error': 'Unknown module'}, 400 - if module_parameters is None and len( - module.module_arguments.keys()) > 0: - return {'error': 'Missing module parameters'}, 400 - for m_args in module.module_arguments: - if job_config.job_type in m_args.get("job_types", []) and \ - m_args.get("required") and \ - m_args.get('name') not in module_parameters.keys(): - return {'error': "Missing required module parameter '" - f"{m_args.get('name')}' for job type " - f"{job_config.job_type}"}, 400 + if module is not None: + if module_parameters is None and len( + module.module_arguments.keys()) > 0: + return {'error': 'Missing module parameters'}, 400 + for m_args in module.module_arguments: + if job_config.job_type in m_args.get("job_types", []) and \ + m_args.get("required") and \ + m_args.get('name') not in module_parameters.keys(): + return {'error': "Missing required module parameter '" + f"{m_args.get('name')}' for job type " + f"{job_config.job_type}"}, 400 if not await SeriesManager.add_series(data): return {'error': 'Something went wrong when adding the series. \ @@ -221,24 +220,21 @@ async def resp_update_series(cls, series_name, data): Returns: dict: dict with data """ - required_fields = ['config'] - if not all(required_field in data - for required_field in required_fields): - return {'error': 'Something went wrong when updating the series. \ - Missing required fields'}, 400 - series_config = SeriesConfigModel(**data.get('config')) + try: + series_config = SeriesConfigModel(**data.get('config')) + except Exception as e: + return {'error': 'Invalid series config', 'message': str(e)}, 400 for job_config in list(series_config.job_config.values()): module_parameters = job_config.module_params module = await EnodoModuleManager.get_module(job_config.module) - if module is None: - return {'error': 'Unknown module'}, 400 - if module_parameters is None and len( - module.module_arguments.keys()) > 0: - return {'error': 'Missing required fields'}, 400 - for key in module.module_arguments: - if key not in module_parameters.keys(): - return {'error': f'Missing required field {key}'}, 400 + if module is not None: + if module_parameters is None and len( + module.module_arguments.keys()) > 0: + return {'error': 'Missing required fields'}, 400 + for key in module.module_arguments: + if key not in module_parameters.keys(): + return {'error': f'Missing required field {key}'}, 400 series = await SeriesManager.get_series(series_name) if series is None: diff --git a/requirements.txt b/requirements.txt index 69e5f55..d0dd023 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,8 @@ aiohttp==3.8.1 aiojobs==1.0.0 aiohttp_basicauth==1.0.0 aiohttp_cors==0.7.0 -jinja2==3.1.0 -psutil==5.9.0 -python-enodo==0.2.15 +jinja2==3.1.2 +python-enodo==0.2.18 python-socketio==5.5.2 qpack==0.0.19 siridb-connector==2.0.8 diff --git a/server.py b/server.py index cfd5887..3b3ae01 100644 --- a/server.py +++ b/server.py @@ -107,7 +107,6 @@ async def start_up(self): await EnodoEventManager.load_from_disk() await EnodoModuleManager.async_setup( SocketIoHandler.internal_updates_enodo_modules_subscribers) - await EnodoModuleManager.load_from_disk() scheduler = ServerState.scheduler self._watch_series_task = await scheduler.spawn(self.watch_series()) @@ -202,6 +201,13 @@ async def _check_for_jobs(self, series, series_name): # Check if base analysis has already run if series.base_analysis_status() == JOB_STATUS_NONE: base_analysis_job = series.base_analysis_job + module = await EnodoModuleManager.get_module( + base_analysis_job.module) + if module is None: + series.state.set_job_check_status( + base_analysis_job.config_name, + "Unknown module") + return await EnodoJobManager.create_job( base_analysis_job.config_name, series_name) # Only continue if base analysis has finished @@ -249,8 +255,6 @@ async def watch_series(self): except Exception as e: logging.error( f"Something went wrong when trying to create new job") - import traceback - traceback.print_exc() logging.debug( f"Corresponding error: {e}, " f'exception class: {e.__class__.__name__}') @@ -264,7 +268,6 @@ async def _save_to_disk(): await SeriesManager.save_to_disk() await EnodoJobManager.save_to_disk() await EnodoEventManager.save_to_disk() - await EnodoModuleManager.save_to_disk() async def save_to_disk(self): """Save configs to disk on a set interval