Live update config file without restarting (RFC)

README.md

@@ -271,6 +271,7 @@ To make deployment with docker easier, most of the important configuration optio
  - FLATHUNTER_GOOGLE_CLOUD_PROJECT_ID - the Google Cloud Project ID, for Google Cloud deployments
  - FLATHUNTER_VERBOSE_LOG - set to any value to enable verbose logging
  - FLATHUNTER_LOOP_PERIOD_SECONDS - a number in seconds for the crawling interval
+ - FLATHUNTER_LOOP_REFRESH_CONFIG - set to any value to enable live editing config file (refresh on each loop)
  - FLATHUNTER_MESSAGE_FORMAT - a format string for the notification messages, where `#CR#` will be replaced by newline
  - FLATHUNTER_NOTIFIERS - a comma-separated list of notifiers to enable (e.g. `telegram,mattermost,slack`)
  - FLATHUNTER_TELEGRAM_BOT_TOKEN - the token for the Telegram notifier

config.yaml.dist

@@ -10,6 +10,7 @@
 loop:
     active: yes
     sleeping_time: 600
+    refresh_config: no
 
 # Location of the Database to store already seen offerings
 # Defaults to the current directory

flathunt.py

@@ -22,6 +22,44 @@
 __status__ = "Production"
 
 
+def check_config(config):
+    logger.info("Checking config for errors")
+    # check config
+    notifiers = config.notifiers()
+    if 'mattermost' in notifiers \
+            and not config.mattermost_webhook_url():
+        logger.error("No Mattermost webhook configured. Starting like this would be pointless...")
+        return
+    if 'telegram' in notifiers:
+        if not config.telegram_bot_token():
+            logger.error(
+                "No Telegram bot token configured. Starting like this would be pointless..."
+            )
+            return
+        if len(config.telegram_receiver_ids()) == 0:
+            logger.warning("No Telegram receivers configured - nobody will get notifications.")
+    if 'apprise' in notifiers \
+            and not config.get('apprise', {}):
+        logger.error("No apprise url configured. Starting like this would be pointless...")
+        return
+    if 'slack' in notifiers \
+            and not config.slack_webhook_url():
+        logger.error("No Slack webhook url configured. Starting like this would be pointless...")
+        return
+
+    if len(config.target_urls()) == 0:
+        logger.error("No URLs configured. Starting like this would be pointless...")
+        return
+
+    return True
+
+
+def get_heartbeat_instructions(args, config):
+    # get heartbeat instructions
+    heartbeat_interval = args.heartbeat
+    heartbeat = Heartbeat(config, heartbeat_interval)
+    return heartbeat
+
 def launch_flat_hunt(config, heartbeat: Heartbeat):
     """Starts the crawler / notification loop"""
     id_watch = IdMaintainer(f'{config.database_location()}/processed_ids.db')
@@ -41,6 +79,36 @@ def launch_flat_hunt(config, heartbeat: Heartbeat):
         counter += 1
         counter = heartbeat.send_heartbeat(counter)
         time.sleep(config.loop_period_seconds())
+
+        if config.loop_refresh_config():
+            args = parse()
+            config_handle = args.config
+            if config_handle is not None:
+                new_config = Config(filename=config_handle.name, silent=True)
+
+                if config.last_modified_time is None or new_config.last_modified_time is None:
+                    logger.warning("Could not compare last modification time of config file."
+                                 "Keeping the old configuration")
+                elif config.last_modified_time < new_config.last_modified_time:
+                    if not check_config(new_config):
+                        logger.warning("Config changed but new config had errors. Keeping old config")
+                    else:
+                        config = new_config
+                        # setup logging
+                        configure_logging(config)
+
+                        # initialize search plugins for config
+                        config.init_searchers()
+
+                        id_watch = IdMaintainer(f'{new_config.database_location()}/processed_ids.db')
+
+                        time_from = dtime.fromisoformat(new_config.loop_pause_from())
+                        time_till = dtime.fromisoformat(new_config.loop_pause_till())
+
+                        wait_during_period(time_from, time_till)
+
+                        hunter = Hunter(new_config, id_watch)
+
         hunter.hunt_flats()
 
 
@@ -60,31 +128,7 @@ def main():
     # initialize search plugins for config
     config.init_searchers()
 
-    # check config
-    notifiers = config.notifiers()
-    if 'mattermost' in notifiers \
-            and not config.mattermost_webhook_url():
-        logger.error("No Mattermost webhook configured. Starting like this would be pointless...")
-        return
-    if 'telegram' in notifiers:
-        if not config.telegram_bot_token():
-            logger.error(
-                "No Telegram bot token configured. Starting like this would be pointless..."
-            )
-            return
-        if len(config.telegram_receiver_ids()) == 0:
-            logger.warning("No Telegram receivers configured - nobody will get notifications.")
-    if 'apprise' in notifiers \
-            and not config.get('apprise', {}):
-        logger.error("No apprise url configured. Starting like this would be pointless...")
-        return
-    if 'slack' in notifiers \
-            and not config.slack_webhook_url():
-        logger.error("No Slack webhook url configured. Starting like this would be pointless...")
-        return
-
-    if len(config.target_urls()) == 0:
-        logger.error("No URLs configured. Starting like this would be pointless...")
+    if not check_config(config):
         return
 
     # get heartbeat instructions

flathunter/config.py

@@ -46,6 +46,8 @@
     FLATHUNTER_VERBOSE_LOG = _read_env("FLATHUNTER_VERBOSE_LOG")
     FLATHUNTER_LOOP_PERIOD_SECONDS = _read_env(
         "FLATHUNTER_LOOP_PERIOD_SECONDS")
+    FLATHUNTER_LOOP_REFRESH_CONFIG = _read_env(
+        "FLATHUNTER_LOOP_REFRESH_CONFIG")
     FLATHUNTER_LOOP_PAUSE_FROM = _read_env("FLATHUNTER_LOOP_PAUSE_FROM")
     FLATHUNTER_LOOP_PAUSE_TILL = _read_env("FLATHUNTER_LOOP_PAUSE_TILL")
     FLATHUNTER_MESSAGE_FORMAT = _read_env("FLATHUNTER_MESSAGE_FORMAT")
@@ -106,6 +108,12 @@
         self.config = config
         self.__searchers__ = []
         self.check_deprecated()
+        if filename := self.config.get("filename"):
+            self.last_modified_time: Optional[float] = self.get_last_modified_time(filename)
+        else:
+            self.last_modified_time: Optional[float] = None
+
+
 
     def __iter__(self):
         """Emulate dictionary"""
@@ -129,6 +137,14 @@
             VrmImmo(self)
         ]
 
+    def get_last_modified_time(self, filename: str) -> Optional[float]:
+        """Gets the time the config file was last modified at the time of initialization"""
+        try:
+            return os.path.getmtime(filename)
+        except OSError as e:
+            logger.error(e)
+            return None
+
     def check_deprecated(self):
         """Notifies user of deprecated config items"""
         captcha_config = self.config.get("captcha")
@@ -209,6 +225,10 @@
         """Return true if flathunter should be crawling in a loop"""
         return self._read_yaml_path('loop.active', False)
 
+    def loop_refresh_config(self):
+        """Return true if flathunter should refresh the config for every loop"""
+        return self._read_yaml_path('loop.refresh_config', False)
+
     def loop_period_seconds(self):
         """Number of seconds to wait between crawls when looping"""
         return self._read_yaml_path('loop.sleeping_time', 60 * 10)
@@ -404,16 +424,18 @@
     environment variable overrides
     """
 
-    def __init__(self, filename=None):
+    def __init__(self, filename=None, silent=False):
         if filename is None and Env.FLATHUNTER_TARGET_URLS is None:
             raise ConfigException(
                 "Config file loaction must be specified, or FLATHUNTER_TARGET_URLS must be set")
         if filename is not None:
-            logger.info("Using config path %s", filename)
+            if not silent:
+                logger.info("Using config path %s", filename)
             if not os.path.exists(filename):
                 raise ConfigException("No config file found at location %s")
             with open(filename, encoding="utf-8") as file:
                 config = yaml.safe_load(file)
+                config["filename"] = filename
         else:
             config = {}
         super().__init__(config)
@@ -444,6 +466,11 @@
             return int(Env.FLATHUNTER_LOOP_PERIOD_SECONDS)
         return super().loop_period_seconds()
 
+    def loop_refresh_config(self):
+        if Env.FLATHUNTER_LOOP_REFRESH_CONFIG is not None:
+            return True
+        return super().loop_refresh_config()
+
     def loop_pause_from(self):
         if Env.FLATHUNTER_LOOP_PAUSE_FROM is not None:
             return str(Env.FLATHUNTER_LOOP_PAUSE_FROM)