Merge branch 'master' into bug/1897809

Change-Id: Ib36cf72fe20aa2b46a80f2d71bcd1400fd70fb8e

.github/workflows/tox.yaml

@@ -0,0 +1,27 @@
+name: Python package
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox tox-gh-actions
+    - name: Lint with tox
+      run: tox -e pep8
+    - name: Test with tox
+      run: tox -e py${{ matrix.python-version }}

requirements.txt

@@ -1,3 +1,7 @@
+# pin lxml < 4.6.3 for py35 as no wheels exist for 4.6.3 (deprecated platform)
+# This is necessary for Xenial builders
+# BUG: https://github.com/openstack-charmers/zaza-openstack-tests/issues/530
+lxml<4.6.3
 aiounittest
 async_generator
 boto3
@@ -29,6 +33,7 @@ python-ceilometerclient
 python-cinderclient
 python-glanceclient
 python-heatclient
+python-ironicclient
 python-keystoneclient
 python-manilaclient
 python-neutronclient
@@ -43,3 +48,8 @@ paramiko
 sphinx
 sphinxcontrib-asyncio
 git+https://github.com/openstack-charmers/zaza#egg=zaza
+
+# Newer versions require a Rust compiler to build, see
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+cryptography<3.4

setup.py

@@ -28,7 +28,12 @@
     'futurist<2.0.0',
     'async_generator',
     'boto3',
-    'cryptography',
+
+    # Newer versions require a Rust compiler to build, see
+    # * https://github.com/openstack-charmers/zaza/issues/421
+    # * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+    'cryptography<3.4',
+
     'dnspython',
     'hvac<0.7.0',
     'jinja2',
@@ -44,6 +49,7 @@
     'python-barbicanclient>=4.0.1,<5.0.0',
     'python-designateclient>=1.5,<3.0.0',
     'python-heatclient<2.0.0',
+    'python-ironicclient',
     'python-glanceclient<3.0.0',
     'python-keystoneclient<3.22.0',
     'python-manilaclient<2.0.0',

tox.ini

@@ -1,6 +1,22 @@
 [tox]
 envlist = pep8, py3
 skipsdist = True
+# NOTE: Avoid build/test env pollution by not enabling sitepackages.
+sitepackages = False
+# NOTE: Avoid false positives by not skipping missing interpreters.
+skip_missing_interpreters = False
+# NOTES:
+# * We avoid the new dependency resolver by pinning pip < 20.3, see
+#   https://github.com/pypa/pip/issues/9187
+# * Pinning dependencies requires tox >= 3.2.0, see
+#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
+# * It is also necessary to pin virtualenv as a newer virtualenv would still
+#   lead to fetching the latest pip in the func* tox targets, see
+#   https://stackoverflow.com/a/38133283
+requires = pip < 20.3
+           virtualenv < 20.0
+# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
+minversion = 3.2.0
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
@@ -14,6 +30,26 @@ commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {t
 basepython = python3
 deps = -r{toxinidir}/requirements.txt
 
+[testenv:py3.5]
+basepython = python3.5
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.6]
+basepython = python3.6
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.7]
+basepython = python3.7
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.8]
+basepython = python3.8
+deps = -r{toxinidir}/requirements.txt
+
+[testenv:py3.9]
+basepython = python3.9
+deps = -r{toxinidir}/requirements.txt
+
 [testenv:pep8]
 basepython = python3
 deps = -r{toxinidir}/requirements.txt

unit_tests/__init__.py

@@ -11,3 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import sys
+import unittest.mock as mock
+
+sys.modules['zaza.utilities.maas'] = mock.MagicMock()

unit_tests/charm_tests/test_utils.py

@@ -97,7 +97,7 @@ def test_config_change(self):
             self.set_application_config.reset_mock()
         self.assertFalse(self.set_application_config.called)
         self.reset_application_config.assert_called_once_with(
-            'anApp', alterna_config.keys(), model_name='aModel')
+            'anApp', list(alterna_config.keys()), model_name='aModel')
         self.wait_for_application_states.assert_has_calls([
             mock.call(model_name='aModel', states={}),
             mock.call(model_name='aModel', states={}),
@@ -125,6 +125,66 @@ def test_config_change(self):
         self.assertFalse(self.wait_for_agent_status.called)
         self.assertFalse(self.wait_for_application_states.called)
 
+    def test_separate_non_string_config(self):
+        intended_cfg_keys = ['foo2', 'foo3', 'foo4', 'foo5']
+        current_config_mock = {
+            'foo2': None,
+            'foo3': 'old_bar3',
+            'foo4': None,
+            'foo5': 'old_bar5',
+        }
+        self.patch_target('config_current')
+        self.config_current.return_value = current_config_mock
+        non_string_type_keys = ['foo2', 'foo3', 'foo4']
+        expected_result_filtered = {
+            'foo3': 'old_bar3',
+            'foo5': 'old_bar5',
+        }
+        expected_result_special = {
+            'foo2': None,
+            'foo4': None,
+        }
+        current, non_string = (
+            self.target.config_current_separate_non_string_type_keys(
+                non_string_type_keys, intended_cfg_keys, 'application_name')
+        )
+
+        self.assertEqual(expected_result_filtered, current)
+        self.assertEqual(expected_result_special, non_string)
+
+        self.config_current.assert_called_once_with(
+            'application_name', intended_cfg_keys)
+
+    def test_separate_special_config_None_params(self):
+        current_config_mock = {
+            'foo1': 'old_bar1',
+            'foo2': None,
+            'foo3': 'old_bar3',
+            'foo4': None,
+            'foo5': 'old_bar5',
+        }
+        self.patch_target('config_current')
+        self.config_current.return_value = current_config_mock
+        non_string_type_keys = ['foo2', 'foo3', 'foo4']
+        expected_result_filtered = {
+            'foo1': 'old_bar1',
+            'foo3': 'old_bar3',
+            'foo5': 'old_bar5',
+        }
+        expected_result_special = {
+            'foo2': None,
+            'foo4': None,
+        }
+        current, non_string = (
+            self.target.config_current_separate_non_string_type_keys(
+                non_string_type_keys)
+        )
+
+        self.assertEqual(expected_result_filtered, current)
+        self.assertEqual(expected_result_special, non_string)
+
+        self.config_current.assert_called_once_with(None, None)
+
 
 class TestOpenStackBaseTest(ut_utils.BaseTestCase):
 

unit_tests/utilities/test_utilities.py

@@ -0,0 +1,188 @@
+# Copyright 2021 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import unit_tests.utils as ut_utils
+
+import zaza.openstack.utilities as utilities
+
+
+class SomeException(Exception):
+    pass
+
+
+class SomeException2(Exception):
+    pass
+
+
+class SomeException3(Exception):
+    pass
+
+
+class TestObjectRetrierWraps(ut_utils.BaseTestCase):
+
+    def test_object_wrap(self):
+
+        class A:
+
+            def func(self, a, b=1):
+                return a + b
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a)
+        self.assertEqual(wrapped_a.func(3), 4)
+
+    def test_object_multilevel_wrap(self):
+
+        class A:
+
+            def f1(self, a, b):
+                return a * b
+
+        class B:
+
+            @property
+            def f2(self):
+
+                return A()
+
+        b = B()
+        wrapped_b = utilities.ObjectRetrierWraps(b)
+        self.assertEqual(wrapped_b.f2.f1(5, 6), 30)
+
+    def test_object_wrap_number(self):
+
+        class A:
+
+            class_a = 5
+
+            def __init__(self):
+                self.instance_a = 10
+
+            def f1(self, a, b):
+                return a * b
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a)
+        self.assertEqual(wrapped_a.class_a, 5)
+        self.assertEqual(wrapped_a.instance_a, 10)
+
+    @mock.patch("time.sleep")
+    def test_object_wrap_exception(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        # retry on a specific exception
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, retry_exceptions=[SomeException])
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
+        # also retry on any exception if none specified
+        wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=1)
+        mock_sleep.reset_mock()
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_called_once_with(5)
+
+        # no retry if exception isn't listed.
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, retry_exceptions=[SomeException2])
+        mock_sleep.reset_mock()
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        mock_sleep.assert_not_called()
+
+    @mock.patch("time.sleep")
+    def test_log_called(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        mock_log = mock.Mock()
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=1, log=mock_log)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+
+        # there should be two calls; one for the single retry and one for the
+        # failure.
+        self.assertEqual(mock_log.call_count, 2)
+
+    @mock.patch("time.sleep")
+    def test_back_off_maximum(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=3, backoff=2)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+        # Note third call hits maximum wait time of 15.
+        mock_sleep.assert_has_calls([mock.call(5),
+                                     mock.call(10),
+                                     mock.call(15)])
+
+    @mock.patch("time.sleep")
+    def test_total_wait(self, mock_sleep):
+
+        class A:
+
+            def func(self):
+                raise SomeException()
+
+        a = A()
+        wrapped_a = utilities.ObjectRetrierWraps(
+            a, num_retries=3, total_wait=9)
+        with self.assertRaises(SomeException):
+            wrapped_a.func()
+        # Note only two calls, as total wait is 9, so a 3rd retry would exceed
+        # that.
+        mock_sleep.assert_has_calls([mock.call(5),
+                                     mock.call(5)])
+
+    @mock.patch("time.sleep")
+    def test_retry_on_connect_failure(self, mock_sleep):
+
+        class A:
+
+            def func1(self):
+                raise SomeException()
+
+            def func2(self):
+                raise utilities.ConnectFailure()
+
+        a = A()
+        wrapped_a = utilities.retry_on_connect_failure(a, num_retries=2)
+        with self.assertRaises(SomeException):
+            wrapped_a.func1()
+        mock_sleep.assert_not_called()
+        with self.assertRaises(utilities.ConnectFailure):
+            wrapped_a.func2()
+        mock_sleep.assert_has_calls([mock.call(5)])