From b6dc44d30385402a2abe8bdb47965c97a59b132a Mon Sep 17 00:00:00 2001 From: Juergen Brendel Date: Tue, 29 Aug 2017 16:33:36 +1200 Subject: [PATCH] End of watch thread correctly detected. A new attempt to start an etcd connection is made. Fixes #9. --- vpcrouter_romana_plugin/__init__.py | 2 +- vpcrouter_romana_plugin/romana.py | 30 ++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/vpcrouter_romana_plugin/__init__.py b/vpcrouter_romana_plugin/__init__.py index 103ae47..8b5b9a6 100644 --- a/vpcrouter_romana_plugin/__init__.py +++ b/vpcrouter_romana_plugin/__init__.py @@ -15,4 +15,4 @@ """ -__version__ = "1.1.9" +__version__ = "1.1.10" diff --git a/vpcrouter_romana_plugin/romana.py b/vpcrouter_romana_plugin/romana.py index 3c8a2c0..d47a2b7 100644 --- a/vpcrouter_romana_plugin/romana.py +++ b/vpcrouter_romana_plugin/romana.py @@ -49,8 +49,9 @@ def __init__(self, *args, **kwargs): self.etcd_latest_raw_time = None self.etcd_connect_time = None - self.watch_id = None # used for etcd APIv3 - self.watch_thread_v2 = None # used for etcd APIv2 + self.watch_id = None # used for etcd APIv3 + self.watch_thread_v2 = None # used for etcd APIv2 + self.watch_broken = False super(Romana, self).__init__(*args, **kwargs) @@ -186,12 +187,21 @@ def watch_loop_v2(self): thread. """ - # By the time we get here, an update may have happened. So, get the - # latest index from the latest result and start our watch there, - # setting the value so that the first watch immediately returns and we - # can send a route spec update. - res = self.etcd.get(self.key) - next_index = res.etcd_index # First watch immediately finds this + try: + # By the time we get here, an update may have happened. So, get the + # latest index from the latest result and start our watch there, + # setting the value so that the first watch immediately returns and + # we can send a route spec update. + res = self.etcd.get(self.key) + next_index = res.etcd_index # First watch immediately finds this + except Exception as e: + # If the etcd isn't healthy, or our data isn't there, then we need + # to end this thread and try again. We use the watch_broken flag to + # indicate failure of this thread. + logging.warning("Romana watcher plugin: Cannot start watch loop: " + "%s" % str(e)) + self.watch_broken = True + return while True: try: @@ -296,10 +306,12 @@ def watch_etcd(self): while self.keep_running: self.etcd = None + self.watch_broken = False self.establish_etcd_connection_and_watch() # Slowly loop as long as the connection status is fine. - while self.etcd_check_status() and self.keep_running: + while self.etcd_check_status() and self.keep_running \ + and not self.watch_broken: time.sleep(self.connect_check_time) logging.warning("Romana watcher plugin: Lost etcd connection.")