diff --git a/requirements.txt b/requirements.txt
index edc52ec5e..80ec7a12a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 pyparsing<3.0.0  # pin for aodhclient which is held for py35
 aiounittest
 async_generator
+kubernetes<18.0.0; python_version < '3.6' # pined, as juju uses kubernetes
 juju
 juju_wait
 PyYAML>=3.0
@@ -19,6 +20,7 @@ Jinja2>=2.6  # BSD License (3 clause)
 six>=1.9.0
 dnspython>=1.12.0
 psutil>=1.1.1,<2.0.0
+oslo.config<6.9.0;python_version < '3.6'  # pin for py3.5 support
 oslo.context<3.0.0;python_version < '3.6'  # pin for py3.5 support
 osprofiler<3.0.0;python_version < '3.6'  # pin for py3.5 support
 python-openstackclient>=3.14.0
diff --git a/tests/bundles/third.yaml b/tests/bundles/third.yaml
new file mode 100644
index 000000000..06a08d670
--- /dev/null
+++ b/tests/bundles/third.yaml
@@ -0,0 +1,5 @@
+applications:
+  ubuntu:
+    charm: cs:ubuntu
+    num_units: 10
+
diff --git a/tests/tests.yaml b/tests/tests.yaml
index 2856bbc88..eee96d9af 100644
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@@ -29,3 +29,4 @@ tests_options:
   force_deploy:
     first
     second
+    third
diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py
index 266efd76c..015f85aa9 100644
--- a/unit_tests/test_zaza_model.py
+++ b/unit_tests/test_zaza_model.py
@@ -578,8 +578,12 @@ def test_get_unit_from_name(self):
 
     def test_get_app_ips(self):
         self.patch_object(model, 'get_juju_model', return_value='mname')
-        self.patch_object(model, 'get_units')
-        self.get_units.return_value = self.units
+        self.patch_object(model, 'async_get_units')
+
+        async def mock_async_aget_units(*args, **kwargs):
+            return self.units
+
+        self.async_get_units.side_effect = mock_async_aget_units
         self.assertEqual(model.get_app_ips('model', 'app'), ['ip1', 'ip2'])
 
     def test_run_on_unit(self):
diff --git a/zaza/charm_tests/libjuju/__init__.py b/zaza/charm_tests/libjuju/__init__.py
index 480703596..3f727d603 100644
--- a/zaza/charm_tests/libjuju/__init__.py
+++ b/zaza/charm_tests/libjuju/__init__.py
@@ -13,4 +13,3 @@
 # limitations under the License.
 
 """Collection of regression tests checking zaza/libjuju integration."""
-
diff --git a/zaza/charm_tests/libjuju/tests.py b/zaza/charm_tests/libjuju/tests.py
index 3e1e84b04..844e03413 100644
--- a/zaza/charm_tests/libjuju/tests.py
+++ b/zaza/charm_tests/libjuju/tests.py
@@ -39,4 +39,3 @@ def test_get_app_ips(self):
         ips = zaza.model.get_app_ips('ubuntu')
         for ip in ips:
             self.assertIsNotNone(ip)
-
diff --git a/zaza/model.py b/zaza/model.py
index 6076f623a..4e6fe65cc 100644
--- a/zaza/model.py
+++ b/zaza/model.py
@@ -696,9 +696,10 @@ async def async_get_app_ips(application_name, model_name=None):
     :returns: List of ip addresses
     :rtype: [str, str,...]
     """
-    return [await u.get_public_address()
-            for u in await async_get_units(
-                application_name, model_name=model_name)]
+    addresses = []
+    for u in await async_get_units(application_name, model_name=model_name):
+        addresses.append(await u.get_public_address())
+    return addresses
 
 
 get_app_ips = sync_wrapper(async_get_app_ips)