Fix race conditions in test suite

mongo_connector/util.py

@@ -61,7 +61,7 @@ def long_to_bson_ts(val):
 def retry_until_ok(func, *args, **kwargs):
     """Retry code block until it succeeds.
 
-    If it does not succeed in 60 attempts, the function re-raises any
+    If it does not succeed in 120 attempts, the function re-raises any
     error the function raised on its last attempt.
 
     """

tests/test_oplog_manager_sharded.py

@@ -30,7 +30,7 @@
 from mongo_connector.locking_dict import LockingDict
 from mongo_connector.oplog_manager import OplogThread
 from mongo_connector.util import retry_until_ok
-from tests import unittest
+from tests import SkipTest, unittest
 from tests.setup_cluster import ShardedCluster
 from tests.util import assert_soon, close_client
 
@@ -421,7 +421,13 @@ def test_rollback(self):
             lambda: shard1_secondary_admin.command("isMaster")["ismaster"])
 
         # Insert another document. This will be rolled back later
-        retry_until_ok(db_main.insert_one, {"i": 1})
+        def cond():
+            try:
+                db_main.insert_one({"i": 1})
+            except:
+                pass
+            return db_main.find_one({"i": 1})
+        retry_until_ok(cond)
         db_secondary1 = self.shard1_secondary_conn["test"]["mcsharded"]
         db_secondary2 = self.shard2_secondary_conn["test"]["mcsharded"]
         self.assertEqual(db_secondary1.count(), 2)
@@ -636,7 +642,10 @@ def move_chunk():
         for op in operations["inprog"]:
             if op.get("query", {}).get("moveChunk"):
                 opid = op["opid"]
-        self.assertNotEqual(opid, None, "could not find moveChunk operation")
+
+        if opid is None:
+            raise SkipTest("could not find moveChunk operation, cannot test "
+                           "failed moveChunk")
         # Kill moveChunk with the opid
 
         if self.mongos_conn.server_info()['versionArray'][:3] >= [3, 1, 2]:

tests/util.py

@@ -21,7 +21,12 @@
 def wait_for(condition, max_tries=60):
     """Wait for a condition to be true up to a maximum number of tries
     """
-    while not condition() and max_tries > 1:
+    cond = False
+    while not cond and max_tries > 1:
+        try:
+            cond = condition()
+        except Exception:
+            pass
         time.sleep(1)
         max_tries -= 1
     return condition()