From 1a985060f44e7476d64b3b97d35b9fbcc4cb2bb2 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Mon, 15 Jan 2024 11:57:36 -0600 Subject: [PATCH 1/3] only emit connection error messages during xread calls if the connection is supposed to be open - this prevents the messages from being printed when a timeout occurs, which causes the connection to be closed and any pending xread calls to fail (when this happened users were confused into thinking that something has broken with roadblock instead of realizing it was a timeout issue) --- roadblock.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/roadblock.py b/roadblock.py index 371a1a3..3debeb6 100644 --- a/roadblock.py +++ b/roadblock.py @@ -1636,8 +1636,12 @@ def run_it(self): self.roadblock_uuid + "__bus__" + self.my_id: personal_last_msg_id }, block = 0) except redis.exceptions.ConnectionError as con_error: - self.logger.error("%s", con_error) - self.logger.error("Bus read failed due to connection error!") + if self.con_pool_state: + self.logger.error("%s", con_error) + self.logger.error("Bus read failed due to connection error!") + else: + self.logger.debug("%s", con_error) + self.logger.debug("Bus read failed because the connection has been closed") except redis.exceptions.TimeoutError as con_error: self.logger.error("%s", con_error) self.logger.error("Bus read failed due to a timeout error!") From d669f9edf43ce99f6512273534373530440ba658 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Mon, 15 Jan 2024 13:02:14 -0600 Subject: [PATCH 2/3] set the return code as early as possible during a timeout - this allows other parts of the code (which may be running in parallel) to see this as soon as possible --- roadblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roadblock.py b/roadblock.py index 3debeb6..7be5a1a 100644 --- a/roadblock.py +++ b/roadblock.py @@ -1252,11 +1252,12 @@ def do_heartbeat_signal(self): def do_timeout(self): '''Handle a roadblock timeout event''' + self.rc = self.RC_TIMEOUT + self.logger.critical("Roadblock failed with timeout") self.timeout_internals() - self.rc = self.RC_TIMEOUT return self.rc def timeout_signal_handler(self, signum, frame): From aac39a102e3a67a0d7f30cd5b996d3e5d4201b70 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Mon, 15 Jan 2024 13:02:55 -0600 Subject: [PATCH 3/3] clear out the msgs array before each xread - if the xread fails we want an empty array, not the previous arrays contents --- roadblock.py | 1 + 1 file changed, 1 insertion(+) diff --git a/roadblock.py b/roadblock.py index 7be5a1a..086d37f 100644 --- a/roadblock.py +++ b/roadblock.py @@ -1623,6 +1623,7 @@ def run_it(self): self.logger.debug("self.rc != 0 --> breaking") break + msgs = [] try: if self.roadblock_role == "follower": msgs = self.redcon.xread(streams = {