From 4c07278303699824ec8883e0b5b460403d989a1e Mon Sep 17 00:00:00 2001 From: Alex O Date: Tue, 6 Feb 2024 19:01:53 +0100 Subject: [PATCH 1/6] Implement feature to live update config file --- flathunt.py | 94 ++++++++++++++++++++++++++++++++------------ flathunter/config.py | 26 +++++++++++- 2 files changed, 93 insertions(+), 27 deletions(-) diff --git a/flathunt.py b/flathunt.py index f5e9d112..01880c6e 100644 --- a/flathunt.py +++ b/flathunt.py @@ -22,6 +22,44 @@ __status__ = "Production" +def check_config(config): + logger.info("Checking config for errors") + # check config + notifiers = config.notifiers() + if 'mattermost' in notifiers \ + and not config.mattermost_webhook_url(): + logger.error("No Mattermost webhook configured. Starting like this would be pointless...") + return + if 'telegram' in notifiers: + if not config.telegram_bot_token(): + logger.error( + "No Telegram bot token configured. Starting like this would be pointless..." + ) + return + if len(config.telegram_receiver_ids()) == 0: + logger.warning("No Telegram receivers configured - nobody will get notifications.") + if 'apprise' in notifiers \ + and not config.get('apprise', {}): + logger.error("No apprise url configured. Starting like this would be pointless...") + return + if 'slack' in notifiers \ + and not config.slack_webhook_url(): + logger.error("No Slack webhook url configured. Starting like this would be pointless...") + return + + if len(config.target_urls()) == 0: + logger.error("No URLs configured. Starting like this would be pointless...") + return + + return True + + +def get_heartbeat_instructions(args, config): + # get heartbeat instructions + heartbeat_interval = args.heartbeat + heartbeat = Heartbeat(config, heartbeat_interval) + return heartbeat + def launch_flat_hunt(config, heartbeat: Heartbeat): """Starts the crawler / notification loop""" id_watch = IdMaintainer(f'{config.database_location()}/processed_ids.db') @@ -41,6 +79,36 @@ def launch_flat_hunt(config, heartbeat: Heartbeat): counter += 1 counter = heartbeat.send_heartbeat(counter) time.sleep(config.loop_period_seconds()) + + if config.loop_refresh_config(): + args = parse() + config_handle = args.config + if config_handle is not None: + new_config = Config(filename=config_handle.name, silent=True) + + if None in (config.last_modified_time, new_config.last_modified_time): + logger.warning("Could not compare last modification time of config file." + "Keeping the old configuration") + elif config.last_modified_time < new_config.last_modified_time: + if not check_config(new_config): + logger.warning("Config changed but new config had errors. Keeping old config") + else: + config = new_config + # setup logging + configure_logging(config) + + # initialize search plugins for config + config.init_searchers() + + id_watch = IdMaintainer(f'{new_config.database_location()}/processed_ids.db') + + time_from = dtime.fromisoformat(new_config.loop_pause_from()) + time_till = dtime.fromisoformat(new_config.loop_pause_till()) + + wait_during_period(time_from, time_till) + + hunter = Hunter(new_config, id_watch) + hunter.hunt_flats() @@ -60,31 +128,7 @@ def main(): # initialize search plugins for config config.init_searchers() - # check config - notifiers = config.notifiers() - if 'mattermost' in notifiers \ - and not config.mattermost_webhook_url(): - logger.error("No Mattermost webhook configured. Starting like this would be pointless...") - return - if 'telegram' in notifiers: - if not config.telegram_bot_token(): - logger.error( - "No Telegram bot token configured. Starting like this would be pointless..." - ) - return - if len(config.telegram_receiver_ids()) == 0: - logger.warning("No Telegram receivers configured - nobody will get notifications.") - if 'apprise' in notifiers \ - and not config.get('apprise', {}): - logger.error("No apprise url configured. Starting like this would be pointless...") - return - if 'slack' in notifiers \ - and not config.slack_webhook_url(): - logger.error("No Slack webhook url configured. Starting like this would be pointless...") - return - - if len(config.target_urls()) == 0: - logger.error("No URLs configured. Starting like this would be pointless...") + if not check_config(config): return # get heartbeat instructions diff --git a/flathunter/config.py b/flathunter/config.py index d82a4a9b..a3eb1027 100644 --- a/flathunter/config.py +++ b/flathunter/config.py @@ -46,6 +46,8 @@ class Env: FLATHUNTER_VERBOSE_LOG = _read_env("FLATHUNTER_VERBOSE_LOG") FLATHUNTER_LOOP_PERIOD_SECONDS = _read_env( "FLATHUNTER_LOOP_PERIOD_SECONDS") + FLATHUNTER_LOOP_REFRESH_CONFIG = _read_env( + "FLATHUNTER_LOOP_REFRESH_CONFIG") FLATHUNTER_LOOP_PAUSE_FROM = _read_env("FLATHUNTER_LOOP_PAUSE_FROM") FLATHUNTER_LOOP_PAUSE_TILL = _read_env("FLATHUNTER_LOOP_PAUSE_TILL") FLATHUNTER_MESSAGE_FORMAT = _read_env("FLATHUNTER_MESSAGE_FORMAT") @@ -106,6 +108,7 @@ def __init__(self, config=None): self.config = config self.__searchers__ = [] self.check_deprecated() + self.last_modified_time: Optional[float] = self.get_last_modified_time(self.config.get("filename")) def __iter__(self): """Emulate dictionary""" @@ -129,6 +132,14 @@ def init_searchers(self): VrmImmo(self) ] + def get_last_modified_time(self, filename: str) -> Optional[float]: + """Gets the time the config file was last modified at the time of initialization""" + try: + return os.path.getmtime(filename) + except Exception as e: + logger.error(e) + return None + def check_deprecated(self): """Notifies user of deprecated config items""" captcha_config = self.config.get("captcha") @@ -209,6 +220,10 @@ def loop_is_active(self): """Return true if flathunter should be crawling in a loop""" return self._read_yaml_path('loop.active', False) + def loop_refresh_config(self): + """Return true if flathunter should refresh the config for every loop""" + return self._read_yaml_path('loop.refresh_config', False) + def loop_period_seconds(self): """Number of seconds to wait between crawls when looping""" return self._read_yaml_path('loop.sleeping_time', 60 * 10) @@ -404,16 +419,18 @@ class Config(CaptchaEnvironmentConfig): # pylint: disable=too-many-public-metho environment variable overrides """ - def __init__(self, filename=None): + def __init__(self, filename=None, silent=False): if filename is None and Env.FLATHUNTER_TARGET_URLS is None: raise ConfigException( "Config file loaction must be specified, or FLATHUNTER_TARGET_URLS must be set") if filename is not None: - logger.info("Using config path %s", filename) + if not silent: + logger.info("Using config path %s", filename) if not os.path.exists(filename): raise ConfigException("No config file found at location %s") with open(filename, encoding="utf-8") as file: config = yaml.safe_load(file) + config["filename"] = filename else: config = {} super().__init__(config) @@ -444,6 +461,11 @@ def loop_period_seconds(self): return int(Env.FLATHUNTER_LOOP_PERIOD_SECONDS) return super().loop_period_seconds() + def loop_refresh_config(self): + if Env.FLATHUNTER_LOOP_REFRESH_CONFIG is not None: + return str(Env.FLATHUNTER_LOOP_REFRESH_CONFIG) + return super().loop_refresh_config() + def loop_pause_from(self): if Env.FLATHUNTER_LOOP_PAUSE_FROM is not None: return str(Env.FLATHUNTER_LOOP_PAUSE_FROM) From af90b87e35cd967fefbdc080c940d77c88066475 Mon Sep 17 00:00:00 2001 From: Alex O Date: Tue, 6 Feb 2024 19:29:04 +0100 Subject: [PATCH 2/6] added refresh_config settings --- config.yaml.dist | 1 + 1 file changed, 1 insertion(+) diff --git a/config.yaml.dist b/config.yaml.dist index 655b90f3..5da66256 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -10,6 +10,7 @@ loop: active: yes sleeping_time: 600 + refresh_config: no # Location of the Database to store already seen offerings # Defaults to the current directory From 73ee4c3b71c1edfadbd618ca725e0a3ef3acc1d9 Mon Sep 17 00:00:00 2001 From: Alex O Date: Tue, 6 Feb 2024 19:37:41 +0100 Subject: [PATCH 3/6] added env value and explanation to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9358c234..29b934e7 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,7 @@ To make deployment with docker easier, most of the important configuration optio - FLATHUNTER_GOOGLE_CLOUD_PROJECT_ID - the Google Cloud Project ID, for Google Cloud deployments - FLATHUNTER_VERBOSE_LOG - set to any value to enable verbose logging - FLATHUNTER_LOOP_PERIOD_SECONDS - a number in seconds for the crawling interval + - FLATHUNTER_LOOP_REFRESH_CONFIG - set to any value to enable live editing config file (refresh on each loop) - FLATHUNTER_MESSAGE_FORMAT - a format string for the notification messages, where `#CR#` will be replaced by newline - FLATHUNTER_NOTIFIERS - a comma-separated list of notifiers to enable (e.g. `telegram,mattermost,slack`) - FLATHUNTER_TELEGRAM_BOT_TOKEN - the token for the Telegram notifier From 607188b3525fe9c5151947799e5dae86b0d71463 Mon Sep 17 00:00:00 2001 From: Alex O Date: Tue, 6 Feb 2024 19:38:39 +0100 Subject: [PATCH 4/6] added correct return value when given env variable --- flathunter/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flathunter/config.py b/flathunter/config.py index 5cbcbd49..34b2f43f 100644 --- a/flathunter/config.py +++ b/flathunter/config.py @@ -463,7 +463,7 @@ def loop_period_seconds(self): def loop_refresh_config(self): if Env.FLATHUNTER_LOOP_REFRESH_CONFIG is not None: - return str(Env.FLATHUNTER_LOOP_REFRESH_CONFIG) + return True return super().loop_refresh_config() def loop_pause_from(self): From 5024cf83444ad6d68a979c0ec13fb045888cab12 Mon Sep 17 00:00:00 2001 From: Alex O Date: Tue, 6 Feb 2024 20:11:17 +0100 Subject: [PATCH 5/6] fix pyright errors --- flathunt.py | 2 +- flathunter/config.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/flathunt.py b/flathunt.py index 01880c6e..06683810 100644 --- a/flathunt.py +++ b/flathunt.py @@ -86,7 +86,7 @@ def launch_flat_hunt(config, heartbeat: Heartbeat): if config_handle is not None: new_config = Config(filename=config_handle.name, silent=True) - if None in (config.last_modified_time, new_config.last_modified_time): + if config.last_modified_time is None or new_config.last_modified_time is None: logger.warning("Could not compare last modification time of config file." "Keeping the old configuration") elif config.last_modified_time < new_config.last_modified_time: diff --git a/flathunter/config.py b/flathunter/config.py index 34b2f43f..bc110a44 100644 --- a/flathunter/config.py +++ b/flathunter/config.py @@ -108,7 +108,12 @@ def __init__(self, config=None): self.config = config self.__searchers__ = [] self.check_deprecated() - self.last_modified_time: Optional[float] = self.get_last_modified_time(self.config.get("filename")) + if filename := self.config.get("filename"): + self.last_modified_time: Optional[float] = self.get_last_modified_time(filename) + else: + self.last_modified_time: Optional[float] = None + + def __iter__(self): """Emulate dictionary""" From ab742929be4b59e999af251e3e27026c45eb50f7 Mon Sep 17 00:00:00 2001 From: Alex O Date: Tue, 6 Feb 2024 20:14:27 +0100 Subject: [PATCH 6/6] fix lint errors --- flathunter/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flathunter/config.py b/flathunter/config.py index bc110a44..ded1d595 100644 --- a/flathunter/config.py +++ b/flathunter/config.py @@ -141,7 +141,7 @@ def get_last_modified_time(self, filename: str) -> Optional[float]: """Gets the time the config file was last modified at the time of initialization""" try: return os.path.getmtime(filename) - except Exception as e: + except OSError as e: logger.error(e) return None