Increase test timeouts (#1046)

* increase timeouts

* add sleep for eventhub listener start

* drop eventhub timeout to 15 minutes

* increase timeouts, make service_helper session scope

* remove extra sleep

dev_utils/dev_utils/service_helper.py

@@ -64,7 +64,7 @@ async def wait_for_eventhub_arrival(self, message_id, timeout=60):
             timeout,
         )
 
-    async def get_next_reported_patch_arrival(self, block=True, timeout=20):
+    async def get_next_reported_patch_arrival(self, block=True, timeout=240):
         return await self._event_loop.run_in_executor(
             self._executor,
             self._inner_object.get_next_reported_patch_arrival,

dev_utils/dev_utils/service_helper_sync.py

@@ -155,7 +155,7 @@ def send_c2d(self, payload, properties):
             raise TypeError("sending C2D to modules is not supported")
         self._registry_manager.send_c2d_message(self.device_id, payload, properties)
 
-    def wait_for_eventhub_arrival(self, message_id, timeout=600):
+    def wait_for_eventhub_arrival(self, message_id, timeout=900):
         def get_event(inner_message_id):
             with self.cv:
                 arrivals = self.incoming_eventhub_events
@@ -191,7 +191,7 @@ def get_event(inner_message_id):
                 else:
                     self.cv.wait()
 
-    def get_next_reported_patch_arrival(self, block=True, timeout=20):
+    def get_next_reported_patch_arrival(self, block=True, timeout=240):
         try:
             return self.incoming_patch_queue.get(block=block, timeout=timeout)
         except queue.Empty:

tests/e2e/iothub_e2e/aio/conftest.py

@@ -56,7 +56,7 @@ def pytest_sessionfinish(session, exitstatus):
             print("-----------------------------------")
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
 def event_loop():
     loop = asyncio.get_event_loop()
     yield loop
@@ -100,7 +100,7 @@ async def client(brand_new_client):
     yield client
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
 async def service_helper(event_loop, executor):
     service_helper = ServiceHelper(
         iothub_connection_string=test_env.IOTHUB_CONNECTION_STRING,
@@ -109,7 +109,6 @@ async def service_helper(event_loop, executor):
         event_loop=event_loop,
         executor=executor,
     )
-    await asyncio.sleep(3)
     yield service_helper
 
     logger.info("----------------------------")

tests/e2e/iothub_e2e/aio/test_twin_stress.py

@@ -70,13 +70,15 @@ async def test_stress_serial_reported_property_updates(
             received = False
             while not received:
                 received_patch = await service_helper.get_next_reported_patch_arrival()
-                if received_patch[const.REPORTED][const.TEST_CONTENT] == patch[const.TEST_CONTENT]:
+                if (
+                    const.REPORTED in received_patch
+                    and received_patch[const.REPORTED][const.TEST_CONTENT]
+                    == patch[const.TEST_CONTENT]
+                ):
                     received = True
                 else:
                     logger.info(
-                        "Wrong patch received. Expecting {}, got {}".format(
-                            received_patch[const.REPORTED], patch
-                        )
+                        "Wrong patch received. Expecting {}, got {}".format(patch, received_patch)
                     )
 
         leak_tracker.check_for_leaks()
@@ -129,20 +131,25 @@ async def test_stress_parallel_reported_property_updates(
                 received_test_content = received_patch[const.REPORTED][const.TEST_CONTENT] or {}
                 logger.info("received {}".format(received_test_content))
 
-                for key in received_test_content.keys():
-                    logger.info("Received {} = {}".format(key, received_test_content[key]))
-                    if key in props:
-                        if received_test_content[key] == props[key]:
-                            logger.info("Key {} received as expected.".format(key))
-                            # Set the value to None so we know that it's been received
-                            props[key] = None
-                            count_received += 1
-                        else:
-                            logger.info(
-                                "Ignoring unexpected value for key {}. Received = {}, expected = {}".format(
-                                    key, received_test_content[key], props[key]
+                if isinstance(received_test_content, dict):
+                    # We check to make sure received_test_content is a dict because it may be
+                    # a string left over from a previous test case.
+                    # This can happen if if the tests are running fast and the reported
+                    # property updates are being processed slowly.
+                    for key in received_test_content.keys():
+                        logger.info("Received {} = {}".format(key, received_test_content[key]))
+                        if key in props:
+                            if received_test_content[key] == props[key]:
+                                logger.info("Key {} received as expected.".format(key))
+                                # Set the value to None so we know that it's been received
+                                props[key] = None
+                                count_received += 1
+                            else:
+                                logger.info(
+                                    "Ignoring unexpected value for key {}. Received = {}, expected = {}".format(
+                                        key, received_test_content[key], props[key]
+                                    )
                                 )
-                            )
 
         leak_tracker.check_for_leaks()
 

tests/e2e/iothub_e2e/conftest.py

@@ -41,7 +41,7 @@ def transport():
     return test_config.config.transport
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
 def executor():
     return concurrent.futures.ThreadPoolExecutor()
 

tests/e2e/iothub_e2e/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-timeout=60
+timeout=120
 testdox_format=plaintext
 junit_logging=all
 junit_family=xunit2

tests/e2e/iothub_e2e/sync/conftest.py

@@ -50,7 +50,7 @@ def client(brand_new_client):
     yield client
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
 def service_helper():
     service_helper = ServiceHelperSync(
         iothub_connection_string=test_env.IOTHUB_CONNECTION_STRING,