Merge pull request #10 from romana/issue-9-watch-reconnect

End of watch thread correctly detected.

vpcrouter_romana_plugin/__init__.py

@@ -15,4 +15,4 @@
 
 """
 
-__version__ = "1.1.9"
+__version__ = "1.1.10"

vpcrouter_romana_plugin/romana.py

@@ -49,8 +49,9 @@ def __init__(self, *args, **kwargs):
         self.etcd_latest_raw_time = None
         self.etcd_connect_time    = None
 
-        self.watch_id             = None  # used for etcd APIv3
-        self.watch_thread_v2      = None  # used for etcd APIv2
+        self.watch_id             = None   # used for etcd APIv3
+        self.watch_thread_v2      = None   # used for etcd APIv2
+        self.watch_broken         = False
 
         super(Romana, self).__init__(*args, **kwargs)
 
@@ -186,12 +187,21 @@ def watch_loop_v2(self):
         thread.
 
         """
-        # By the time we get here, an update may have happened. So, get the
-        # latest index from the latest result and start our watch there,
-        # setting the value so that the first watch immediately returns and we
-        # can send a route spec update.
-        res        = self.etcd.get(self.key)
-        next_index = res.etcd_index   # First watch immediately finds this
+        try:
+            # By the time we get here, an update may have happened. So, get the
+            # latest index from the latest result and start our watch there,
+            # setting the value so that the first watch immediately returns and
+            # we can send a route spec update.
+            res        = self.etcd.get(self.key)
+            next_index = res.etcd_index   # First watch immediately finds this
+        except Exception as e:
+            # If the etcd isn't healthy, or our data isn't there, then we need
+            # to end this thread and try again. We use the watch_broken flag to
+            # indicate failure of this thread.
+            logging.warning("Romana watcher plugin: Cannot start watch loop: "
+                            "%s" % str(e))
+            self.watch_broken = True
+            return
 
         while True:
             try:
@@ -296,10 +306,12 @@ def watch_etcd(self):
         while self.keep_running:
             self.etcd = None
 
+            self.watch_broken = False
             self.establish_etcd_connection_and_watch()
 
             # Slowly loop as long as the connection status is fine.
-            while self.etcd_check_status() and self.keep_running:
+            while self.etcd_check_status() and self.keep_running \
+                                and not self.watch_broken:
                 time.sleep(self.connect_check_time)
 
             logging.warning("Romana watcher plugin: Lost etcd connection.")