diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml new file mode 100644 index 000000000..754ce297f --- /dev/null +++ b/.github/workflows/tox.yaml @@ -0,0 +1,27 @@ +name: Python package + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Lint with tox + run: tox -e pep8 + - name: Test with tox + run: tox -e py${{ matrix.python-version }} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8b15edc8a..adec5bcec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# pin lxml < 4.6.3 for py35 as no wheels exist for 4.6.3 (deprecated platform) +# This is necessary for Xenial builders +# BUG: https://github.com/openstack-charmers/zaza-openstack-tests/issues/530 +lxml<4.6.3 aiounittest async_generator boto3 @@ -29,6 +33,7 @@ python-ceilometerclient python-cinderclient python-glanceclient python-heatclient +python-ironicclient python-keystoneclient python-manilaclient python-neutronclient @@ -43,3 +48,8 @@ paramiko sphinx sphinxcontrib-asyncio git+https://github.com/openstack-charmers/zaza#egg=zaza + +# Newer versions require a Rust compiler to build, see +# * https://github.com/openstack-charmers/zaza/issues/421 +# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html +cryptography<3.4 diff --git a/setup.py b/setup.py index f4e31f8cd..fb4377d31 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,12 @@ 'futurist<2.0.0', 'async_generator', 'boto3', - 'cryptography', + + # Newer versions require a Rust compiler to build, see + # * https://github.com/openstack-charmers/zaza/issues/421 + # * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html + 'cryptography<3.4', + 'dnspython', 'hvac<0.7.0', 'jinja2', @@ -44,6 +49,7 @@ 'python-barbicanclient>=4.0.1,<5.0.0', 'python-designateclient>=1.5,<3.0.0', 'python-heatclient<2.0.0', + 'python-ironicclient', 'python-glanceclient<3.0.0', 'python-keystoneclient<3.22.0', 'python-manilaclient<2.0.0', diff --git a/tox.ini b/tox.ini index 7d67e9f8a..aea173f9a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,22 @@ [tox] envlist = pep8, py3 skipsdist = True +# NOTE: Avoid build/test env pollution by not enabling sitepackages. +sitepackages = False +# NOTE: Avoid false positives by not skipping missing interpreters. +skip_missing_interpreters = False +# NOTES: +# * We avoid the new dependency resolver by pinning pip < 20.3, see +# https://github.com/pypa/pip/issues/9187 +# * Pinning dependencies requires tox >= 3.2.0, see +# https://tox.readthedocs.io/en/latest/config.html#conf-requires +# * It is also necessary to pin virtualenv as a newer virtualenv would still +# lead to fetching the latest pip in the func* tox targets, see +# https://stackoverflow.com/a/38133283 +requires = pip < 20.3 + virtualenv < 20.0 +# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci +minversion = 3.2.0 [testenv] setenv = VIRTUAL_ENV={envdir} @@ -14,6 +30,26 @@ commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {t basepython = python3 deps = -r{toxinidir}/requirements.txt +[testenv:py3.5] +basepython = python3.5 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.6] +basepython = python3.6 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.7] +basepython = python3.7 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.8] +basepython = python3.8 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.9] +basepython = python3.9 +deps = -r{toxinidir}/requirements.txt + [testenv:pep8] basepython = python3 deps = -r{toxinidir}/requirements.txt diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index 8203d13e1..03c4879a7 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -11,3 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import sys +import unittest.mock as mock + +sys.modules['zaza.utilities.maas'] = mock.MagicMock() diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index ffb7f83f1..c1e8166a3 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -97,7 +97,7 @@ def test_config_change(self): self.set_application_config.reset_mock() self.assertFalse(self.set_application_config.called) self.reset_application_config.assert_called_once_with( - 'anApp', alterna_config.keys(), model_name='aModel') + 'anApp', list(alterna_config.keys()), model_name='aModel') self.wait_for_application_states.assert_has_calls([ mock.call(model_name='aModel', states={}), mock.call(model_name='aModel', states={}), @@ -125,6 +125,66 @@ def test_config_change(self): self.assertFalse(self.wait_for_agent_status.called) self.assertFalse(self.wait_for_application_states.called) + def test_separate_non_string_config(self): + intended_cfg_keys = ['foo2', 'foo3', 'foo4', 'foo5'] + current_config_mock = { + 'foo2': None, + 'foo3': 'old_bar3', + 'foo4': None, + 'foo5': 'old_bar5', + } + self.patch_target('config_current') + self.config_current.return_value = current_config_mock + non_string_type_keys = ['foo2', 'foo3', 'foo4'] + expected_result_filtered = { + 'foo3': 'old_bar3', + 'foo5': 'old_bar5', + } + expected_result_special = { + 'foo2': None, + 'foo4': None, + } + current, non_string = ( + self.target.config_current_separate_non_string_type_keys( + non_string_type_keys, intended_cfg_keys, 'application_name') + ) + + self.assertEqual(expected_result_filtered, current) + self.assertEqual(expected_result_special, non_string) + + self.config_current.assert_called_once_with( + 'application_name', intended_cfg_keys) + + def test_separate_special_config_None_params(self): + current_config_mock = { + 'foo1': 'old_bar1', + 'foo2': None, + 'foo3': 'old_bar3', + 'foo4': None, + 'foo5': 'old_bar5', + } + self.patch_target('config_current') + self.config_current.return_value = current_config_mock + non_string_type_keys = ['foo2', 'foo3', 'foo4'] + expected_result_filtered = { + 'foo1': 'old_bar1', + 'foo3': 'old_bar3', + 'foo5': 'old_bar5', + } + expected_result_special = { + 'foo2': None, + 'foo4': None, + } + current, non_string = ( + self.target.config_current_separate_non_string_type_keys( + non_string_type_keys) + ) + + self.assertEqual(expected_result_filtered, current) + self.assertEqual(expected_result_special, non_string) + + self.config_current.assert_called_once_with(None, None) + class TestOpenStackBaseTest(ut_utils.BaseTestCase): diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py new file mode 100644 index 000000000..6af6089b3 --- /dev/null +++ b/unit_tests/utilities/test_utilities.py @@ -0,0 +1,188 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import unit_tests.utils as ut_utils + +import zaza.openstack.utilities as utilities + + +class SomeException(Exception): + pass + + +class SomeException2(Exception): + pass + + +class SomeException3(Exception): + pass + + +class TestObjectRetrierWraps(ut_utils.BaseTestCase): + + def test_object_wrap(self): + + class A: + + def func(self, a, b=1): + return a + b + + a = A() + wrapped_a = utilities.ObjectRetrierWraps(a) + self.assertEqual(wrapped_a.func(3), 4) + + def test_object_multilevel_wrap(self): + + class A: + + def f1(self, a, b): + return a * b + + class B: + + @property + def f2(self): + + return A() + + b = B() + wrapped_b = utilities.ObjectRetrierWraps(b) + self.assertEqual(wrapped_b.f2.f1(5, 6), 30) + + def test_object_wrap_number(self): + + class A: + + class_a = 5 + + def __init__(self): + self.instance_a = 10 + + def f1(self, a, b): + return a * b + + a = A() + wrapped_a = utilities.ObjectRetrierWraps(a) + self.assertEqual(wrapped_a.class_a, 5) + self.assertEqual(wrapped_a.instance_a, 10) + + @mock.patch("time.sleep") + def test_object_wrap_exception(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + # retry on a specific exception + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, retry_exceptions=[SomeException]) + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # also retry on any exception if none specified + wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=1) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # no retry if exception isn't listed. + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, retry_exceptions=[SomeException2]) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_not_called() + + @mock.patch("time.sleep") + def test_log_called(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + mock_log = mock.Mock() + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, log=mock_log) + with self.assertRaises(SomeException): + wrapped_a.func() + + # there should be two calls; one for the single retry and one for the + # failure. + self.assertEqual(mock_log.call_count, 2) + + @mock.patch("time.sleep") + def test_back_off_maximum(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=3, backoff=2) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note third call hits maximum wait time of 15. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(10), + mock.call(15)]) + + @mock.patch("time.sleep") + def test_total_wait(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=3, total_wait=9) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note only two calls, as total wait is 9, so a 3rd retry would exceed + # that. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(5)]) + + @mock.patch("time.sleep") + def test_retry_on_connect_failure(self, mock_sleep): + + class A: + + def func1(self): + raise SomeException() + + def func2(self): + raise utilities.ConnectFailure() + + a = A() + wrapped_a = utilities.retry_on_connect_failure(a, num_retries=2) + with self.assertRaises(SomeException): + wrapped_a.func1() + mock_sleep.assert_not_called() + with self.assertRaises(utilities.ConnectFailure): + wrapped_a.func2() + mock_sleep.assert_has_calls([mock.call(5)]) diff --git a/unit_tests/utilities/test_zaza_utilities_ceph.py b/unit_tests/utilities/test_zaza_utilities_ceph.py index ea718b789..e6ac337c0 100644 --- a/unit_tests/utilities/test_zaza_utilities_ceph.py +++ b/unit_tests/utilities/test_zaza_utilities_ceph.py @@ -118,7 +118,7 @@ def test_get_rbd_hash(self): model_name='amodel') def test_pools_from_broker_req(self): - self.patch_object(ceph_utils.zaza_juju, 'get_relation_from_unit') + self.patch_object(ceph_utils.juju_utils, 'get_relation_from_unit') self.get_relation_from_unit.return_value = { 'broker_req': ( '{"api-version": 1, "ops": [' diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 02bd5783b..0434a2be8 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -17,6 +17,8 @@ import io import mock import subprocess +import sys +import unittest import tenacity import unit_tests.utils as ut_utils @@ -191,6 +193,7 @@ def _test_get_overcloud_auth(self, tls_relation=False, ssl_cert=False, self.patch_object(openstack_utils, 'get_application_config_option') self.patch_object(openstack_utils, 'get_keystone_ip') self.patch_object(openstack_utils, "get_current_os_versions") + self.patch_object(openstack_utils, "get_remote_ca_cert_file") self.patch_object(openstack_utils.juju_utils, 'leader_get') if tls_relation: self.patch_object(openstack_utils.model, "scp_from_unit") @@ -204,6 +207,7 @@ def _test_get_overcloud_auth(self, tls_relation=False, ssl_cert=False, self.get_relation_id.return_value = None self.get_application_config_option.return_value = None self.leader_get.return_value = 'openstack' + self.get_remote_ca_cert_file.return_value = None if tls_relation or ssl_cert: port = 35357 transport = 'https' @@ -245,7 +249,8 @@ def _test_get_overcloud_auth(self, tls_relation=False, ssl_cert=False, 'API_VERSION': 3, } if tls_relation: - expect['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + self.get_remote_ca_cert_file.return_value = '/tmp/a.cert' + expect['OS_CACERT'] = '/tmp/a.cert' self.assertEqual(openstack_utils.get_overcloud_auth(), expect) @@ -289,6 +294,15 @@ def test_get_undercloud_keystone_session(self): openstack_utils.get_undercloud_keystone_session() self.get_keystone_session.assert_called_once_with(_auth, verify=None) + def test_get_nova_session_client(self): + session_mock = mock.MagicMock() + self.patch_object(openstack_utils.novaclient_client, "Client") + openstack_utils.get_nova_session_client(session_mock) + self.Client.assert_called_once_with(2, session=session_mock) + self.Client.reset_mock() + openstack_utils.get_nova_session_client(session_mock, version=2.56) + self.Client.assert_called_once_with(2.56, session=session_mock) + def test_get_urllib_opener(self): self.patch_object(openstack_utils.urllib.request, "ProxyHandler") self.patch_object(openstack_utils.urllib.request, "HTTPHandler") @@ -369,12 +383,15 @@ def test__resource_reaches_status_fail(self): 'e01df65a') def test__resource_reaches_status_bespoke(self): + client_mock = mock.MagicMock() resource_mock = mock.MagicMock() - resource_mock.get.return_value = mock.MagicMock(status='readyish') + resource_mock.special_status = 'readyish' + client_mock.get.return_value = resource_mock openstack_utils._resource_reaches_status( - resource_mock, + client_mock, 'e01df65a', - 'readyish') + 'readyish', + resource_attribute='special_status') def test__resource_reaches_status_bespoke_fail(self): resource_mock = mock.MagicMock() @@ -504,7 +521,7 @@ def test_upload_image_to_glance(self): glance_mock.images.upload.assert_called_once_with( '9d1125af', f(), - ) + backend=None) self.resource_reaches_status.assert_called_once_with( glance_mock.images, '9d1125af', @@ -529,7 +546,12 @@ def test_create_image_use_tempdir(self): self.upload_image_to_glance.assert_called_once_with( glance_mock, 'wibbly/c.img', - 'bob') + 'bob', + backend=None, + disk_format='qcow2', + visibility='public', + container_format='bare', + force_import=False) def test_create_image_pass_directory(self): glance_mock = mock.MagicMock() @@ -549,7 +571,12 @@ def test_create_image_pass_directory(self): self.upload_image_to_glance.assert_called_once_with( glance_mock, 'tests/c.img', - 'bob') + 'bob', + backend=None, + disk_format='qcow2', + visibility='public', + container_format='bare', + force_import=False) self.gettempdir.assert_not_called() def test_create_ssh_key(self): @@ -859,7 +886,24 @@ def test_get_current_openstack_release_pair(self): result = openstack_utils.get_current_os_release_pair() self.assertEqual(expected, result) - def test_get_openstack_release(self): + def test_get_current_os_versions(self): + self.patch_object(openstack_utils, "get_openstack_release") + self.patch_object(openstack_utils.generic_utils, "get_pkg_version") + + # Pre-Wallaby scenario where openstack-release package isn't installed + self.get_openstack_release.return_value = None + self.get_pkg_version.return_value = '18.0.0' + expected = {'keystone': 'victoria'} + result = openstack_utils.get_current_os_versions('keystone') + self.assertEqual(expected, result) + + # Wallaby+ scenario where openstack-release package is installed + self.get_openstack_release.return_value = 'wallaby' + expected = {'keystone': 'wallaby'} + result = openstack_utils.get_current_os_versions('keystone') + self.assertEqual(expected, result) + + def test_get_os_release(self): self.patch( 'zaza.openstack.utilities.openstack.get_current_os_release_pair', new_callable=mock.MagicMock(), @@ -888,6 +932,14 @@ def test_get_openstack_release(self): release_comp = xenial_queens > xenial_mitaka self.assertTrue(release_comp) + # Check specifying an application + self._get_os_rel_pair.reset_mock() + self._get_os_rel_pair.return_value = 'xenial_mitaka' + expected = 4 + result = openstack_utils.get_os_release(application='myapp') + self.assertEqual(expected, result) + self._get_os_rel_pair.assert_called_once_with(application='myapp') + def test_get_keystone_api_version(self): self.patch_object(openstack_utils, "get_current_os_versions") self.patch_object(openstack_utils, "get_application_config_option") @@ -903,6 +955,23 @@ def test_get_keystone_api_version(self): self.get_application_config_option.return_value = None self.assertEqual(openstack_utils.get_keystone_api_version(), 3) + def test_get_openstack_release(self): + self.patch_object(openstack_utils.model, "get_units") + self.patch_object(openstack_utils.juju_utils, "remote_run") + + # Test pre-Wallaby behavior where openstack-release pkg isn't installed + self.get_units.return_value = [] + self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby " + + # Test Wallaby+ behavior where openstack-release package is installed + unit1 = mock.MagicMock() + unit1.entity_id = 1 + self.get_units.return_value = [unit1] + self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby " + + result = openstack_utils.get_openstack_release("application", "model") + self.assertEqual(result, "wallaby") + def test_get_project_id(self): # No domain self.patch_object(openstack_utils, "get_keystone_api_version") @@ -1234,34 +1303,230 @@ def test_ngw_present(self): self.get_application.side_effect = KeyError self.assertFalse(openstack_utils.ngw_present()) - def test_configure_gateway_ext_port(self): - # FIXME: this is not a complete unit test for the function as one did - # not exist at all I'm adding this to test one bit and we'll add more - # as we go. + def test_get_charm_networking_data(self): self.patch_object(openstack_utils, 'deprecated_external_networking') self.patch_object(openstack_utils, 'dvr_enabled') self.patch_object(openstack_utils, 'ovn_present') self.patch_object(openstack_utils, 'ngw_present') + self.patch_object(openstack_utils, 'get_ovs_uuids') self.patch_object(openstack_utils, 'get_gateway_uuids') - self.patch_object(openstack_utils, 'get_admin_net') + self.patch_object(openstack_utils, 'get_ovn_uuids') + self.patch_object(openstack_utils.model, 'get_application') self.dvr_enabled.return_value = False self.ovn_present.return_value = False + self.ngw_present.return_value = False + self.get_ovs_uuids.return_value = [] + self.get_gateway_uuids.return_value = [] + self.get_ovn_uuids.return_value = [] + self.get_application.side_effect = KeyError + + with self.assertRaises(RuntimeError): + openstack_utils.get_charm_networking_data() self.ngw_present.return_value = True - self.get_admin_net.return_value = {'id': 'fakeid'} + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS, + ['neutron-gateway'], + mock.ANY, + 'data-port', + {})) + self.dvr_enabled.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR, + ['neutron-gateway', 'neutron-openvswitch'], + mock.ANY, + 'data-port', + {})) + self.ngw_present.return_value = False + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT, + ['neutron-openvswitch'], + mock.ANY, + 'data-port', + {})) + self.dvr_enabled.return_value = False + self.ovn_present.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis'], + mock.ANY, + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'})) + self.get_application.side_effect = None + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis', 'ovn-dedicated-chassis'], + mock.ANY, + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'})) + + def test_get_cacert_absolute_path(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' + self.assertEqual( + openstack_utils.get_cacert_absolute_path('filename'), + '/tmp/default/filename') + + def test_get_cacert(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' + self.patch_object(openstack_utils.os.path, 'exists') + results = { + '/tmp/default/vault_juju_ca_cert.crt': True} + self.exists.side_effect = lambda x: results[x] + self.assertEqual( + openstack_utils.get_cacert(), + '/tmp/default/vault_juju_ca_cert.crt') - novaclient = mock.MagicMock() - neutronclient = mock.MagicMock() + results = { + '/tmp/default/vault_juju_ca_cert.crt': False, + '/tmp/default/keystone_juju_ca_cert.crt': True} + self.assertEqual( + openstack_utils.get_cacert(), + '/tmp/default/keystone_juju_ca_cert.crt') - def _fake_empty_generator(empty=True): - if empty: - return - yield + results = { + '/tmp/default/vault_juju_ca_cert.crt': False, + '/tmp/default/keystone_juju_ca_cert.crt': False} + self.assertIsNone(openstack_utils.get_cacert()) - self.get_gateway_uuids.side_effect = _fake_empty_generator - with self.assertRaises(RuntimeError): - openstack_utils.configure_gateway_ext_port( - novaclient, neutronclient) - # provide a uuid and check that we don't raise RuntimeError - self.get_gateway_uuids.side_effect = ['fake-uuid'] - openstack_utils.configure_gateway_ext_port( - novaclient, neutronclient) + def test_get_remote_ca_cert_file(self): + self.patch_object(openstack_utils.model, 'get_first_unit_name') + self.patch_object( + openstack_utils, + '_get_remote_ca_cert_file_candidates') + self.patch_object(openstack_utils.model, 'scp_from_unit') + self.patch_object(openstack_utils.os.path, 'exists') + self.patch_object(openstack_utils.shutil, 'move') + self.patch_object(openstack_utils.os, 'chmod') + self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile') + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' + enter_mock = mock.MagicMock() + enter_mock.__enter__.return_value.name = 'tempfilename' + self.NamedTemporaryFile.return_value = enter_mock + self.get_first_unit_name.return_value = 'neutron-api/0' + self._get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/ca1.cert'] + self.exists.return_value = True + + openstack_utils.get_remote_ca_cert_file('neutron-api') + self.scp_from_unit.assert_called_once_with( + 'neutron-api/0', + '/tmp/ca1.cert', + 'tempfilename') + self.chmod.assert_called_once_with('/tmp/default/ca1.cert', 0o644) + self.move.assert_called_once_with( + 'tempfilename', '/tmp/default/ca1.cert') + + +class TestAsyncOpenstackUtils(ut_utils.AioTestCase): + + def setUp(self): + super(TestAsyncOpenstackUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + model_mock = mock.MagicMock() + test_mock = mock.MagicMock() + + class AsyncContextManagerMock(test_mock): + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + self.model_mock = model_mock + self.patch_object(openstack_utils.zaza.model, "async_block_until") + + async def _block_until(f, timeout): + # Store the result of the call to _check_ca_present to validate + # tests + self.result = await f() + self.async_block_until.side_effect = _block_until + self.patch('zaza.model.run_in_model', name='_run_in_model') + self._run_in_model.return_value = AsyncContextManagerMock + self._run_in_model().__aenter__.return_value = self.model_mock + + async def test_async_block_until_ca_exists(self): + def _get_action_output(stdout, code, stderr=None): + stderr = stderr or '' + action = mock.MagicMock() + action.data = { + 'results': { + 'Code': code, + 'Stderr': stderr, + 'Stdout': stdout}} + return action + results = { + '/tmp/missing.cert': _get_action_output( + '', + '1', + 'cat: /tmp/missing.cert: No such file or directory'), + '/tmp/good.cert': _get_action_output('CERTIFICATE', '0')} + + async def _run(command, timeout=None): + return results[command.split()[-1]] + self.unit1 = mock.MagicMock() + self.unit2 = mock.MagicMock() + self.unit2.run.side_effect = _run + self.unit1.run.side_effect = _run + self.units = [self.unit1, self.unit2] + _units = mock.MagicMock() + _units.units = self.units + self.model_mock.applications = { + 'keystone': _units + } + self.patch_object( + openstack_utils, + "_async_get_remote_ca_cert_file_candidates") + + # Test a missing cert then a good cert. + self._async_get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/missing.cert', + '/tmp/good.cert'] + await openstack_utils.async_block_until_ca_exists( + 'keystone', + 'CERTIFICATE') + self.assertTrue(self.result) + + # Test a single missing + self._async_get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/missing.cert'] + await openstack_utils.async_block_until_ca_exists( + 'keystone', + 'CERTIFICATE') + self.assertFalse(self.result) + + async def test__async_get_remote_ca_cert_file_candidates(self): + self.patch_object(openstack_utils.zaza.model, "async_get_relation_id") + rel_id_out = { + } + + def _get_relation_id(app, cert_app, model_name, remote_interface_name): + return rel_id_out[cert_app] + self.async_get_relation_id.side_effect = _get_relation_id + + rel_id_out['vault'] = 'certs:1' + r = await openstack_utils._async_get_remote_ca_cert_file_candidates( + 'neutron-api', 'mymodel') + self.assertEqual( + r, + ['/usr/local/share/ca-certificates/vault_juju_ca_cert.crt', + '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt']) + + rel_id_out['vault'] = None + r = await openstack_utils._async_get_remote_ca_cert_file_candidates( + 'neutron-api', 'mymodel') + self.assertEqual( + r, + ['/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt']) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index 4d619c53a..3782926a2 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -44,25 +44,37 @@ def setUp(self): self.patch_object( openstack_upgrade.zaza.model, "get_application_config") + self.patch_object( + openstack_upgrade.zaza.model, + "block_until_all_units_idle") + self.patch_object( + openstack_upgrade, + "block_until_mysql_innodb_cluster_has_rw") def _get_application_config(app, model_name=None): app_config = { - 'ceph-mon': {'verbose': True, 'source': 'old-src'}, - 'neutron-openvswitch': {'verbose': True}, - 'ntp': {'verbose': True}, - 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'ceph-mon': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, + 'neutron-openvswitch': {'verbose': {'value': True}}, + 'ntp': {'verbose': {'value': True}}, + 'percona-cluster': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, 'cinder': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'neutron-api': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'nova-compute': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, + 'mysql-innodb-cluster': { + 'verbose': {'value': True}, + 'source': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': True}}, } return app_config[app] self.get_application_config.side_effect = _get_application_config @@ -74,6 +86,10 @@ def _get_application_config(app, model_name=None): 'subordinate-to': 'nova-compute'}, 'ntp': { # Filter as it has no source option 'charm': 'cs:ntp'}, + 'mysql-innodb-cluster': { + 'charm': 'cs:mysql-innodb-cluster', + 'units': { + 'mysql-innodb-cluster/0': {}}}, 'nova-compute': { 'charm': 'cs:nova-compute', 'units': { @@ -115,7 +131,7 @@ def test_action_unit_upgrade(self): model_name=None, raise_on_failure=True) - def test_action_upgrade_group(self): + def test_action_upgrade_apps(self): self.patch_object(openstack_upgrade, "pause_units") self.patch_object(openstack_upgrade, "action_unit_upgrade") self.patch_object(openstack_upgrade, "resume_units") @@ -127,7 +143,7 @@ def test_action_upgrade_group(self): 'nova-compute': [mock_nova_compute_0], 'cinder': [mock_cinder_1]} self.get_units.side_effect = lambda app, model_name: units[app] - openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder']) + openstack_upgrade.action_upgrade_apps(['nova-compute', 'cinder']) pause_calls = [ mock.call(['cinder-hacluster/0'], model_name=None), mock.call(['nova-compute/0', 'cinder/1'], model_name=None)] @@ -142,6 +158,30 @@ def test_action_upgrade_group(self): mock.call(['cinder-hacluster/0'], model_name=None)] self.resume_units.assert_has_calls(resume_calls, any_order=False) + def test_action_upgrade_apps_mysql_innodb_cluster(self): + """Verify that mysql-innodb-cluster is settled before complete.""" + self.patch_object(openstack_upgrade, "pause_units") + self.patch_object(openstack_upgrade, "action_unit_upgrade") + self.patch_object(openstack_upgrade, "resume_units") + mock_mysql_innodb_cluster_0 = mock.MagicMock() + mock_mysql_innodb_cluster_0.entity_id = 'mysql-innodb-cluster/0' + units = {'mysql-innodb-cluster': [mock_mysql_innodb_cluster_0]} + self.get_units.side_effect = lambda app, model_name: units[app] + openstack_upgrade.action_upgrade_apps(['mysql-innodb-cluster']) + pause_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.pause_units.assert_has_calls(pause_calls, any_order=False) + action_unit_upgrade_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.action_unit_upgrade.assert_has_calls( + action_unit_upgrade_calls, + any_order=False) + resume_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.resume_units.assert_has_calls(resume_calls, any_order=False) + self.block_until_mysql_innodb_cluster_has_rw.assert_called_once_with( + None) + def test_set_upgrade_application_config(self): openstack_upgrade.set_upgrade_application_config( ['neutron-api', 'cinder'], @@ -177,17 +217,23 @@ def test_is_action_upgradable(self): self.assertFalse( openstack_upgrade.is_action_upgradable('percona-cluster')) + def test_is_already_upgraded(self): + self.assertTrue( + openstack_upgrade.is_already_upgraded('cinder', 'old-src')) + self.assertFalse( + openstack_upgrade.is_already_upgraded('cinder', 'new-src')) + def test_run_action_upgrade(self): self.patch_object(openstack_upgrade, "set_upgrade_application_config") - self.patch_object(openstack_upgrade, "action_upgrade_group") - openstack_upgrade.run_action_upgrade( + self.patch_object(openstack_upgrade, "action_upgrade_apps") + openstack_upgrade.run_action_upgrades( ['cinder', 'neutron-api'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) - self.action_upgrade_group.assert_called_once_with( + self.action_upgrade_apps.assert_called_once_with( ['cinder', 'neutron-api'], model_name=None) @@ -196,7 +242,7 @@ def test_run_all_in_one_upgrade(self): self.patch_object( openstack_upgrade.zaza.model, 'block_until_all_units_idle') - openstack_upgrade.run_all_in_one_upgrade( + openstack_upgrade.run_all_in_one_upgrades( ['percona-cluster'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( @@ -207,34 +253,36 @@ def test_run_all_in_one_upgrade(self): self.block_until_all_units_idle.assert_called_once_with() def test_run_upgrade(self): - self.patch_object(openstack_upgrade, "run_all_in_one_upgrade") - self.patch_object(openstack_upgrade, "run_action_upgrade") - openstack_upgrade.run_upgrade( + self.patch_object(openstack_upgrade, "run_all_in_one_upgrades") + self.patch_object(openstack_upgrade, "run_action_upgrades") + openstack_upgrade.run_upgrade_on_apps( ['cinder', 'neutron-api', 'ceph-mon'], 'new-src') - self.run_all_in_one_upgrade.assert_called_once_with( + self.run_all_in_one_upgrades.assert_called_once_with( ['ceph-mon'], 'new-src', model_name=None) - self.run_action_upgrade.assert_called_once_with( + self.run_action_upgrades.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) def test_run_upgrade_tests(self): - self.patch_object(openstack_upgrade, "run_upgrade") + self.patch_object(openstack_upgrade, "run_upgrade_on_apps") self.patch_object(openstack_upgrade, "get_upgrade_groups") - self.get_upgrade_groups.return_value = { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder', 'neutron-api'], - 'Core Identity': ['keystone'], - 'Storage': ['ceph-mon'], - 'sweep_up': ['designate']} + self.get_upgrade_groups.return_value = [ + ('Compute', ['nova-compute']), + ('Control Plane', ['cinder', 'neutron-api']), + ('Core Identity', ['keystone']), + ('Storage', ['ceph-mon']), + ('sweep_up', ['designate'])] openstack_upgrade.run_upgrade_tests('new-src', model_name=None) run_upgrade_calls = [ + mock.call(['nova-compute'], 'new-src', model_name=None), + mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), mock.call(['keystone'], 'new-src', model_name=None), mock.call(['ceph-mon'], 'new-src', model_name=None), - mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), - mock.call(['nova-compute'], 'new-src', model_name=None), - mock.call(['designate'], 'new-src', model_name=None)] - self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False) + mock.call(['designate'], 'new-src', model_name=None), + ] + self.run_upgrade_on_apps.assert_has_calls( + run_upgrade_calls, any_order=False) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 457ce1b98..70ffd3bee 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import mock import sys import unittest @@ -139,28 +138,7 @@ def test_app_config_percona(self): self.assertEqual(expected, config) -class AioTestCase(ut_utils.BaseTestCase): - def __init__(self, methodName='runTest', loop=None): - self.loop = loop or asyncio.get_event_loop() - self._function_cache = {} - super(AioTestCase, self).__init__(methodName=methodName) - - def coroutine_function_decorator(self, func): - def wrapper(*args, **kw): - return self.loop.run_until_complete(func(*args, **kw)) - return wrapper - - def __getattribute__(self, item): - attr = object.__getattribute__(self, item) - if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): - if item not in self._function_cache: - self._function_cache[item] = ( - self.coroutine_function_decorator(attr)) - return self._function_cache[item] - return attr - - -class TestParallelSeriesUpgrade(AioTestCase): +class TestParallelSeriesUpgrade(ut_utils.AioTestCase): def setUp(self): super(TestParallelSeriesUpgrade, self).setUp() if sys.version_info < (3, 6, 0): @@ -182,6 +160,8 @@ def setUp(self): self.model.async_wait_for_unit_idle = mock.AsyncMock() self.async_run_on_machine = mock.AsyncMock() self.model.async_run_on_machine = self.async_run_on_machine + self.model.async_block_until_units_on_machine_are_idle = \ + mock.AsyncMock() @mock.patch.object(upgrade_utils.cl_utils, 'get_class') async def test_run_post_application_upgrade_functions( @@ -492,7 +472,13 @@ async def test_series_upgrade_machine_with_source( mock_remove_confdef_file.assert_called_once_with('1') mock_add_confdef_file.assert_called_once_with('1') - async def test_maybe_pause_things_primary(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_primary(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], @@ -503,7 +489,13 @@ async def test_maybe_pause_things_primary(self): mock.call('app/2', "pause", action_params={}), ]) - async def test_maybe_pause_things_subordinates(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_subordinates(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], @@ -514,7 +506,13 @@ async def test_maybe_pause_things_subordinates(self): mock.call('app-hacluster/2', "pause", action_params={}), ]) - async def test_maybe_pause_things_all(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_all(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], diff --git a/unit_tests/utils.py b/unit_tests/utils.py index 4694d0d9e..8e31f45f9 100644 --- a/unit_tests/utils.py +++ b/unit_tests/utils.py @@ -19,6 +19,7 @@ """Module to provide helper for writing unit tests.""" +import asyncio import contextlib import io import mock @@ -96,3 +97,24 @@ def patch(self, item, return_value=None, name=None, new=None, **kwargs): started.return_value = return_value self._patches_start[name] = started setattr(self, name, started) + + +class AioTestCase(BaseTestCase): + def __init__(self, methodName='runTest', loop=None): + self.loop = loop or asyncio.get_event_loop() + self._function_cache = {} + super(AioTestCase, self).__init__(methodName=methodName) + + def coroutine_function_decorator(self, func): + def wrapper(*args, **kw): + return self.loop.run_until_complete(func(*args, **kw)) + return wrapper + + def __getattribute__(self, item): + attr = object.__getattribute__(self, item) + if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): + if item not in self._function_cache: + self._function_cache[item] = ( + self.coroutine_function_decorator(attr)) + return self._function_cache[item] + return attr diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index e0178dff9..3b966e523 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -25,6 +25,7 @@ import zaza.openstack.configure.guest import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.configure.telemetry as telemetry_utils @@ -70,6 +71,13 @@ def query_aodh_api(self): """Check that aodh api is responding.""" self.aodh_client.alarm.list() + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + @property def services(self): """Return a list of the service that should be running.""" @@ -139,6 +147,26 @@ def test_901_pause_resume(self): logging.info("Testing pause resume") self.query_aodh_api() + def test_902_nrpe_service_checks(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units('aodh') + cmds = [] + if self.release >= self.xenial_ocata: + services = ['aodh-evaluator', 'aodh-notifier', + 'aodh-listener', 'apache2'] + else: + services = ['aodh-api', 'aodh-evaluator', + 'aodh-notifier', 'aodh-listener'] + for check_name in services: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + if ret: + logging.info(ret) + self.assertIsNone(ret, msg=ret) + class AodhServerAlarmTest(test_utils.OpenStackBaseTest): """Test server events trigger Aodh alarms.""" diff --git a/zaza/openstack/charm_tests/ceph/dashboard/__init__.py b/zaza/openstack/charm_tests/ceph/dashboard/__init__.py new file mode 100644 index 000000000..f34c394e5 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ``ceph-dashboard``.""" diff --git a/zaza/openstack/charm_tests/ceph/dashboard/tests.py b/zaza/openstack/charm_tests/ceph/dashboard/tests.py new file mode 100644 index 000000000..e7c8863b3 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/tests.py @@ -0,0 +1,97 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulating `ceph-dashboard` testing.""" + +import collections +import os +import requests + +import zaza +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.utilities.deployment_env as deployment_env + + +class CephDashboardTest(test_utils.BaseCharmTest): + """Class for `ceph-dashboard` tests.""" + + REMOTE_CERT_FILE = ('/usr/local/share/ca-certificates/' + 'vault_ca_cert_dashboard.crt') + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph dashboard tests.""" + super().setUpClass() + cls.application_name = 'ceph-dashboard' + cls.local_ca_cert = cls.collect_ca() + + @classmethod + def collect_ca(cls): + """Collect CA from ceph-dashboard unit.""" + local_ca_cert = os.path.join( + deployment_env.get_tmpdir(), + os.path.basename(cls.REMOTE_CERT_FILE)) + if not os.path.isfile(local_ca_cert): + units = zaza.model.get_units(cls.application_name) + zaza.model.scp_from_unit( + units[0].entity_id, + cls.REMOTE_CERT_FILE, + local_ca_cert) + return local_ca_cert + + def test_dashboard_units(self): + """Check dashboard units are configured correctly.""" + # XXX: Switch to using CA for verification when + # https://bugs.launchpad.net/cloud-archive/+bug/1933410 + # is fix released. + # verify = self.local_ca_cert + verify = False + units = zaza.model.get_units(self.application_name) + rcs = collections.defaultdict(list) + for unit in units: + r = requests.get( + 'https://{}:8443'.format(unit.public_address), + verify=verify, + allow_redirects=False) + rcs[r.status_code].append(unit.public_address) + self.assertEqual(len(rcs[requests.codes.ok]), 1) + self.assertEqual(len(rcs[requests.codes.see_other]), len(units) - 1) + + def create_user(self, username, role='administrator'): + """Create a dashboard user. + + :param username: Username to create. + :type username: str + :param role: Role to grant to user. + :type role: str + :returns: Results from action. + :rtype: juju.action.Action + """ + action = zaza.model.run_action_on_leader( + 'ceph-dashboard', + 'add-user', + action_params={ + 'username': username, + 'role': role}) + return action + + def test_create_user(self): + """Test create user action.""" + test_user = 'marvin' + action = self.create_user(test_user) + self.assertEqual(action.status, "completed") + self.assertTrue(action.data['results']['password']) + action = self.create_user(test_user) + # Action should fail as the user already exists + self.assertEqual(action.status, "failed") diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index a986833a5..0766e4ac1 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -64,26 +64,25 @@ def get_base_ctxt(self): :rtype: Dict """ gw_units = zaza.model.get_units('ceph-iscsi') - primary_gw = gw_units[0] - secondary_gw = gw_units[1] host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True) client_entity_ids = [ u.entity_id for u in zaza.model.get_units('ubuntu')] ctxt = { 'client_entity_ids': sorted(client_entity_ids), 'gw_iqn': self.GW_IQN, - 'gw1_ip': primary_gw.public_address, - 'gw1_hostname': host_names[primary_gw.entity_id], - 'gw1_entity_id': primary_gw.entity_id, - 'gw2_ip': secondary_gw.public_address, - 'gw2_hostname': host_names[secondary_gw.entity_id], - 'gw2_entity_id': secondary_gw.entity_id, 'chap_creds': 'username={chap_username} password={chap_password}', 'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways', 'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts', 'gwcli_disk_dir': '/disks', 'gwcli_client_dir': '{gwcli_hosts_dir}/{client_initiatorname}', } + ctxt['gateway_units'] = [ + { + 'entity_id': u.entity_id, + 'ip': u.public_address, + 'hostname': host_names[u.entity_id]} + for u in zaza.model.get_units('ceph-iscsi')] + ctxt['gw_ip'] = sorted([g['ip'] for g in ctxt['gateway_units']])[0] return ctxt def run_commands(self, unit_name, commands, ctxt): @@ -116,9 +115,8 @@ def create_iscsi_target(self, ctxt): 'ceph-iscsi', 'create-target', action_params={ - 'gateway-units': '{} {}'.format( - ctxt['gw1_entity_id'], - ctxt['gw2_entity_id']), + 'gateway-units': ' '.join([g['entity_id'] + for g in ctxt['gateway_units']]), 'iqn': self.GW_IQN, 'rbd-pool-name': ctxt.get('pool_name', ''), 'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''), @@ -139,7 +137,7 @@ def login_iscsi_target(self, ctxt): base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} ' '--op=update ').format(**ctxt) setup_cmds = [ - 'iscsiadm -m discovery -t st -p {gw1_ip}', + 'iscsiadm -m discovery -t st -p {gw_ip}', base_op_cmd + '-n node.session.auth.authmethod -v CHAP', base_op_cmd + '-n node.session.auth.username -v {chap_username}', base_op_cmd + '-n node.session.auth.password -v {chap_password}', diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index e3f7be1ac..e598a5d72 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -16,6 +16,9 @@ import logging import unittest +import re + +from copy import deepcopy import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -47,3 +50,235 @@ def test_osd_security_checklist(self): expected_passes, expected_failures, expected_to_pass=True) + + +class OsdService: + """Simple representation of ceph-osd systemd service.""" + + def __init__(self, id_): + """ + Init service using its ID. + + e.g.: id_=1 -> ceph-osd@1 + """ + self.id = id_ + self.name = 'ceph-osd@{}'.format(id_) + + +async def async_wait_for_service_status(unit_name, services, target_status, + model_name=None, timeout=2700): + """Wait for all services on the unit to be in the desired state. + + Note: This function emulates the + `zaza.model.async_block_until_service_status` function, but it's using + `systemctl is-active` command instead of `pidof/pgrep` of the original + function. + + :param unit_name: Name of unit to run action on + :type unit_name: str + :param services: List of services to check + :type services: List[str] + :param target_status: State services must be in (stopped or running) + :type target_status: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: Time to wait for status to be achieved + :type timeout: int + """ + async def _check_service(): + services_ok = True + for service in services: + command = r"systemctl is-active '{}'".format(service) + out = await zaza_model.async_run_on_unit( + unit_name, + command, + model_name=model_name, + timeout=timeout) + response = out['Stdout'].strip() + + if target_status == "running" and response == 'active': + continue + elif target_status == "stopped" and response == 'inactive': + continue + else: + services_ok = False + break + + return services_ok + + accepted_states = ('stopped', 'running') + if target_status not in accepted_states: + raise RuntimeError('Invalid target state "{}". Accepted states: ' + '{}'.format(target_status, accepted_states)) + + async with zaza_model.run_in_model(model_name): + await zaza_model.async_block_until(_check_service, timeout=timeout) + + +wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) + + +class ServiceTest(unittest.TestCase): + """ceph-osd systemd service tests.""" + + TESTED_UNIT = 'ceph-osd/0' # This can be any ceph-osd unit in the model + SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') + + def __init__(self, methodName='runTest'): + """Initialize Test Case.""" + super(ServiceTest, self).__init__(methodName) + self._available_services = None + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph service tests.""" + super(ServiceTest, cls).setUpClass() + + def setUp(self): + """Run test setup.""" + # Skip 'service' action tests on systems without systemd + result = zaza_model.run_on_unit(self.TESTED_UNIT, 'which systemctl') + if not result['Stdout']: + raise unittest.SkipTest("'service' action is not supported on " + "systems without 'systemd'. Skipping " + "tests.") + # Note(mkalcok): This counter reset is needed because ceph-osd service + # is limited to 3 restarts per 30 mins which is insufficient + # when running functional tests for 'service' action. This + # limitation is defined in /lib/systemd/system/ceph-osd@.service + # in section [Service] with options 'StartLimitInterval' and + # 'StartLimitBurst' + reset_counter = 'systemctl reset-failed' + zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter) + + def tearDown(self): + """Start ceph-osd services after each test. + + This ensures that the environment is ready for the next tests. + """ + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'osds': 'all'}, + raise_on_failure=True) + + @property + def available_services(self): + """Return list of all ceph-osd services present on the TESTED_UNIT.""" + if self._available_services is None: + self._available_services = self._fetch_osd_services() + return self._available_services + + def _fetch_osd_services(self): + """Fetch all ceph-osd services present on the TESTED_UNIT.""" + service_list = [] + service_list_cmd = 'systemctl list-units --full --all ' \ + '--no-pager -t service' + result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd) + for line in result['Stdout'].split('\n'): + service_name = self.SERVICE_PATTERN.search(line) + if service_name: + service_id = int(service_name.group('service_id')) + service_list.append(OsdService(service_id)) + return service_list + + def test_start_stop_all_by_keyword(self): + """Start and Stop all ceph-osd services using keyword 'all'.""" + service_list = [service.name for service in self.available_services] + + logging.info("Running 'service stop=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT], 'stop', + action_params={'osds': 'all'}) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'osds': 'all'}) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') + + def test_start_stop_all_by_list(self): + """Start and Stop all ceph-osd services using explicit list.""" + service_list = [service.name for service in self.available_services] + service_ids = [str(service.id) for service in self.available_services] + action_params = ','.join(service_ids) + + logging.info("Running 'service stop={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', + action_params={'osds': action_params}) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'osds': action_params}) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') + + def test_stop_specific(self): + """Stop only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + should_run = deepcopy(self.available_services) + to_stop = should_run.pop() + should_run = [service.name for service in should_run] + + logging.info("Running 'service stop={} on {} " + "unit".format(to_stop.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', + action_params={'osds': to_stop.id}) + + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running') + + def test_start_specific(self): + """Start only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + service_names = [service.name for service in self.available_services] + should_stop = deepcopy(self.available_services) + to_start = should_stop.pop() + should_stop = [service.name for service in should_stop] + + # Note: can't stop ceph-osd.target as restarting a single OSD will + # cause this to start all of the OSDs when a single one starts. + logging.info("Stopping all running ceph-osd services") + service_stop_cmd = '; '.join(['systemctl stop {}'.format(service) + for service in service_names]) + zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) + + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') + + logging.info("Running 'service start={} on {} " + "unit".format(to_start.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'osds': to_start.id}) + + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running') + + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 6c2fa1b2c..d8d796776 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -17,6 +17,8 @@ import logging import re +import cinderclient.exceptions as cinder_exceptions + import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model @@ -28,6 +30,129 @@ CIRROS_IMAGE_NAME) +DEFAULT_CINDER_RBD_MIRRORING_MODE = 'pool' + + +def get_cinder_rbd_mirroring_mode(cinder_ceph_app_name='cinder-ceph'): + """Get the RBD mirroring mode for the Cinder Ceph pool. + + :param cinder_ceph_app_name: Cinder Ceph Juju application name. + :type cinder_ceph_app_name: str + :returns: A string representing the RBD mirroring mode. It can be + either 'pool' or 'image'. + :rtype: str + """ + rbd_mirroring_mode_config = zaza.model.get_application_config( + cinder_ceph_app_name).get('rbd-mirroring-mode') + if rbd_mirroring_mode_config: + rbd_mirroring_mode = rbd_mirroring_mode_config.get( + 'value', DEFAULT_CINDER_RBD_MIRRORING_MODE).lower() + else: + rbd_mirroring_mode = DEFAULT_CINDER_RBD_MIRRORING_MODE + + return rbd_mirroring_mode + + +def get_glance_image(glance): + """Get the Glance image object to be used by the Ceph tests. + + It looks for the Cirros Glance image, and it's returned if it's found. + If the Cirros image is not found, it will try and find the Ubuntu + LTS image. + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :returns: Glance image object + :rtype: glanceclient.image + """ + images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if images: + return images[0] + logging.info("Failed to find {} image, falling back to {}".format( + CIRROS_IMAGE_NAME, + LTS_IMAGE_NAME)) + return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + + +def setup_cinder_repl_volume_type(cinder, type_name='repl', + backend_name='cinder-ceph'): + """Set up the Cinder volume replication type. + + :param cinder: Authenticated cinderclient + :type cinder: cinder.Client + :param type_name: Cinder volume type name + :type type_name: str + :param backend_name: Cinder volume backend name with replication enabled. + :type backend_name: str + :returns: Cinder volume type object + :rtype: cinderclient.VolumeType + """ + try: + vol_type = cinder.volume_types.find(name=type_name) + except cinder_exceptions.NotFound: + vol_type = cinder.volume_types.create(type_name) + + vol_type.set_keys(metadata={ + 'volume_backend_name': backend_name, + 'replication_enabled': ' True', + }) + return vol_type + + +# TODO: This function should be incorporated into +# 'zaza.openstack.utilities.openstack.create_volume' helper, once the below +# flakiness comments are addressed. +def create_cinder_volume(cinder, name='zaza', image_id=None, type_id=None): + """Create a new Cinder volume. + + :param cinder: Authenticated cinderclient. + :type cinder: cinder.Client + :param name: Volume name. + :type name: str + :param image_id: Glance image id, if the volume is created from image. + :type image_id: str + :param type_id: Cinder Volume type id, if the volume needs to use an + explicit volume type. + :type type_id: boolean + :returns: Cinder volume + :rtype: :class:`Volume`. + """ + # NOTE(fnordahl): for some reason create volume from image often fails + # when run just after deployment is finished. We should figure out + # why, resolve the underlying issue and then remove this. + # + # We do not use tenacity here as it will interfere with tenacity used + # in ``resource_reaches_status`` + def create_volume(cinder, volume_params, retry=20): + if retry < 1: + return + volume = cinder.volumes.create(**volume_params) + try: + # Note(coreycb): stop_after_attempt is increased because using + # juju storage for ceph-osd backed by cinder on undercloud + # takes longer than the prior method of directory-backed OSD + # devices. + openstack.resource_reaches_status( + cinder.volumes, volume.id, msg='volume', + stop_after_attempt=20) + return volume + except AssertionError: + logging.info('retrying') + volume.delete() + return create_volume(cinder, volume_params, retry=retry - 1) + + volume_params = { + 'size': 8, + 'name': name, + } + if image_id: + volume_params['imageRef'] = image_id + if type_id: + volume_params['volume_type'] = type_id + + return create_volume(cinder, volume_params) + + class CephRBDMirrorBase(test_utils.OpenStackBaseTest): """Base class for ``ceph-rbd-mirror`` tests.""" @@ -35,20 +160,26 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run setup for ``ceph-rbd-mirror`` tests.""" super().setUpClass() + cls.cinder_ceph_app_name = 'cinder-ceph' + cls.test_cinder_volume_name = 'test-cinder-ceph-volume' # get ready for multi-model Zaza cls.site_a_model = cls.site_b_model = zaza.model.get_juju_model() cls.site_b_app_suffix = '-b' - def run_status_action(self, application_name=None, model_name=None): + def run_status_action(self, application_name=None, model_name=None, + pools=[]): """Run status action, decode and return response.""" + action_params = { + 'verbose': True, + 'format': 'json', + } + if len(pools) > 0: + action_params['pools'] = ','.join(pools) result = zaza.model.run_action_on_leader( application_name or self.application_name, 'status', model_name=model_name, - action_params={ - 'verbose': True, - 'format': 'json', - }) + action_params=action_params) return json.loads(result.results['output']) def get_pools(self): @@ -68,10 +199,26 @@ def get_pools(self): model_name=self.site_b_model) return sorted(site_a_pools.keys()), sorted(site_b_pools.keys()) + def get_failover_pools(self): + """Get the failover Ceph pools' names, from both sites. + + If the Cinder RBD mirroring mode is 'image', the 'cinder-ceph' pool + needs to be excluded, since Cinder orchestrates the failover then. + + :returns: Tuple with site-a pools and site-b pools. + :rtype: Tuple[List[str], List[str]] + """ + site_a_pools, site_b_pools = self.get_pools() + if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image': + site_a_pools.remove(self.cinder_ceph_app_name) + site_b_pools.remove(self.cinder_ceph_app_name) + return site_a_pools, site_b_pools + def wait_for_mirror_state(self, state, application_name=None, model_name=None, check_entries_behind_master=False, - require_images_in=[]): + require_images_in=[], + pools=[]): """Wait until all images reach requested state. This function runs the ``status`` action and examines the data it @@ -90,6 +237,9 @@ def wait_for_mirror_state(self, state, application_name=None, :type check_entries_behind_master: bool :param require_images_in: List of pools to require images in :type require_images_in: list of str + :param pools: List of pools to run status on. If this is empty, the + status action will run on all the pools. + :type pools: list of str :returns: True on success, never returns on failure """ rep = re.compile(r'.*entries_behind_master=(\d+)') @@ -97,7 +247,8 @@ def wait_for_mirror_state(self, state, application_name=None, try: # encapsulate in try except to work around LP: #1820976 pool_status = self.run_status_action( - application_name=application_name, model_name=model_name) + application_name=application_name, model_name=model_name, + pools=pools) except KeyError: continue for pool, status in pool_status.items(): @@ -124,6 +275,41 @@ def wait_for_mirror_state(self, state, application_name=None, # all images with state has expected state return True + def setup_test_cinder_volume(self): + """Set up the test Cinder volume into the Ceph RBD mirror environment. + + If the volume already exists, then it's returned. + + Also, if the Cinder RBD mirroring mode is 'image', the volume will + use an explicit volume type with the appropriate replication flags. + Otherwise, it is just a simple Cinder volume using the default backend. + + :returns: Cinder volume + :rtype: :class:`Volume`. + """ + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + + try: + return cinder.volumes.find(name=self.test_cinder_volume_name) + except cinder_exceptions.NotFound: + logging.info("Test Cinder volume doesn't exist. Creating it") + + glance = openstack.get_glance_session_client(session) + image = get_glance_image(glance) + kwargs = { + 'cinder': cinder, + 'name': self.test_cinder_volume_name, + 'image_id': image.id, + } + if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image': + volume_type = setup_cinder_repl_volume_type( + cinder, + backend_name=self.cinder_ceph_app_name) + kwargs['type_id'] = volume_type.id + + return create_cinder_volume(**kwargs) + class CephRBDMirrorTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` tests.""" @@ -195,44 +381,7 @@ def test_cinder_volume_mirrored(self): site B and subsequently comparing the contents we get a full end to end test. """ - session = openstack.get_overcloud_keystone_session() - glance = openstack.get_glance_session_client(session) - cinder = openstack.get_cinder_session_client(session) - - images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) - if images: - image = images[0] - else: - logging.info("Failed to find {} image, falling back to {}".format( - CIRROS_IMAGE_NAME, - LTS_IMAGE_NAME)) - image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] - - # NOTE(fnordahl): for some reason create volume from image often fails - # when run just after deployment is finished. We should figure out - # why, resolve the underlying issue and then remove this. - # - # We do not use tenacity here as it will interfere with tenacity used - # in ``resource_reaches_status`` - def create_volume_from_image(cinder, image, retry=20): - if retry < 1: - return - volume = cinder.volumes.create(8, name='zaza', imageRef=image.id) - try: - # Note(coreycb): stop_after_attempt is increased because using - # juju storage for ceph-osd backed by cinder on undercloud - # takes longer than the prior method of directory-backed OSD - # devices. - openstack.resource_reaches_status( - cinder.volumes, volume.id, msg='volume', - stop_after_attempt=20) - return volume - except AssertionError: - logging.info('retrying') - volume.delete() - return create_volume_from_image(cinder, image, retry=retry - 1) - volume = create_volume_from_image(cinder, image) - + volume = self.setup_test_cinder_volume() site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon', model_name=self.site_a_model), @@ -244,6 +393,8 @@ def create_volume_from_image(cinder, image, retry=20): check_entries_behind_master=True, application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model) + logging.info('Checking the Ceph RBD hashes of the primary and ' + 'the secondary Ceph images') site_b_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon' + self.site_b_app_suffix, model_name=self.site_b_model), @@ -258,102 +409,399 @@ def create_volume_from_image(cinder, image, retry=20): class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` controlled failover tests.""" - def test_fail_over_fall_back(self): - """Validate controlled fail over and fall back.""" - site_a_pools, site_b_pools = self.get_pools() - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror', - 'demote', - model_name=self.site_a_model, - action_params={}) - logging.info(result.results) - n_pools_demoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state('up+unknown', model_name=self.site_a_model) - self.wait_for_mirror_state( - 'up+unknown', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, - 'promote', - model_name=self.site_b_model, - action_params={}) - logging.info(result.results) - n_pools_promoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_b_pools), n_pools_promoted) - self.wait_for_mirror_state( - 'up+replaying', - model_name=self.site_a_model) - self.wait_for_mirror_state( - 'up+stopped', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + def execute_failover_juju_actions(self, + primary_site_app_name, + primary_site_model, + primary_site_pools, + secondary_site_app_name, + secondary_site_model, + secondary_site_pools): + """Execute the failover Juju actions. + + The failover / failback via Juju actions shares the same workflow. The + failback is just a failover with sites in reversed order. + + This function encapsulates the tasks to failover a primary site to + a secondary site: + 1. Demote primary site + 2. Validation of the primary site demotion + 3. Promote secondary site + 4. Validation of the secondary site promotion + + :param primary_site_app_name: Primary site Ceph RBD mirror app name. + :type primary_site_app_name: str + :param primary_site_model: Primary site Juju model name. + :type primary_site_model: str + :param primary_site_pools: Primary site pools. + :type primary_site_pools: List[str] + :param secondary_site_app_name: Secondary site Ceph RBD mirror + app name. + :type secondary_site_app_name: str + :param secondary_site_model: Secondary site Juju model name. + :type secondary_site_model: str + :param secondary_site_pools: Secondary site pools. + :type secondary_site_pools: List[str] + """ + # Check if primary and secondary pools sizes are the same. + self.assertEqual(len(primary_site_pools), len(secondary_site_pools)) + + # Run the 'demote' Juju action against the primary site pools. + logging.info('Demoting {} from model {}.'.format( + primary_site_app_name, primary_site_model)) result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + primary_site_app_name, 'demote', - model_name=self.site_b_model, + model_name=primary_site_model, action_params={ + 'pools': ','.join(primary_site_pools) }) logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the demoted pools count matches the total primary site + # pools count. n_pools_demoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_a_pools), n_pools_demoted) + self.assertEqual(len(primary_site_pools), n_pools_demoted) + + # At this point, both primary and secondary sites are demoted. Validate + # that the Ceph images, from both sites, report 'up+unknown', since + # there isn't a primary site at the moment. + logging.info('Waiting until {} is demoted.'.format( + primary_site_app_name)) self.wait_for_mirror_state( 'up+unknown', - model_name=self.site_a_model) + application_name=primary_site_app_name, + model_name=primary_site_model, + pools=primary_site_pools) self.wait_for_mirror_state( 'up+unknown', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + application_name=secondary_site_app_name, + model_name=secondary_site_model, + pools=secondary_site_pools) + + # Run the 'promote' Juju against the secondary site. + logging.info('Promoting {} from model {}.'.format( + secondary_site_app_name, secondary_site_model)) result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror', + secondary_site_app_name, 'promote', - model_name=self.site_a_model, + model_name=secondary_site_model, action_params={ + 'pools': ','.join(secondary_site_pools) }) logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the promoted pools count matches the total secondary + # site pools count. n_pools_promoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_b_pools), n_pools_promoted) + self.assertEqual(len(secondary_site_pools), n_pools_promoted) + + # Validate that the Ceph images from the newly promoted site + # report 'up+stopped' state (which is reported by primary Ceph images). + logging.info('Waiting until {} is promoted.'.format( + secondary_site_app_name)) self.wait_for_mirror_state( 'up+stopped', - model_name=self.site_a_model) + application_name=secondary_site_app_name, + model_name=secondary_site_model, + pools=secondary_site_pools) + + # Validate that the Ceph images from site-a report 'up+replaying' + # (which is reported by secondary Ceph images). + self.wait_for_mirror_state( + 'up+replaying', + check_entries_behind_master=True, + application_name=primary_site_app_name, + model_name=primary_site_model, + pools=primary_site_pools) + + def test_100_cinder_failover(self): + """Validate controlled failover via the Cinder API. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_100_cinder_failover' since Cinder RBD " + "mirroring mode is {}.".format(cinder_rbd_mirroring_mode)) + return + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + + # Check if the Cinder volume host is available with replication + # enabled. + host = 'cinder@{}'.format(self.cinder_ceph_app_name) + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + self.assertEqual(svc.replication_status, 'enabled') + self.assertEqual(svc.status, 'enabled') + + # Setup the test Cinder volume + volume = self.setup_test_cinder_volume() + + # Check if the volume is properly mirrored + self.wait_for_mirror_state( + 'up+replaying', + check_entries_behind_master=True, + application_name=self.application_name + self.site_b_app_suffix, + model_name=self.site_b_model, + pools=[self.cinder_ceph_app_name]) + + # Execute the Cinder volume failover + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over') + + # Check if the test volume is still available after failover + self.assertEqual(cinder.volumes.get(volume.id).status, 'available') + + def test_101_cinder_failback(self): + """Validate controlled failback via the Cinder API. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + + The test needs to be executed when the Cinder volume host is already + failed-over with the test volume on it. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_101_cinder_failback' since Cinder RBD " + "mirroring mode is {}.".format(cinder_rbd_mirroring_mode)) + return + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + + # Check if the Cinder volume host is already failed-over + host = 'cinder@{}'.format(self.cinder_ceph_app_name) + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + self.assertEqual(svc.replication_status, 'failed-over') + self.assertEqual(svc.status, 'disabled') + + # Check if the test Cinder volume is already present. The method + # 'cinder.volumes.find' raises 404 if the volume is not found. + volume = cinder.volumes.find(name=self.test_cinder_volume_name) + + # Execute the Cinder volume failback + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='default', + target_status='enabled', + target_replication_status='enabled') + + # Check if the test volume is still available after failback + self.assertEqual(cinder.volumes.get(volume.id).status, 'available') + + def test_200_juju_failover(self): + """Validate controlled failover via Juju actions.""" + # Get the Ceph pools needed to failover + site_a_pools, site_b_pools = self.get_failover_pools() + + # Execute the failover Juju actions with the appropriate parameters. + site_b_app_name = self.application_name + self.site_b_app_suffix + self.execute_failover_juju_actions( + primary_site_app_name=self.application_name, + primary_site_model=self.site_a_model, + primary_site_pools=site_a_pools, + secondary_site_app_name=site_b_app_name, + secondary_site_model=self.site_b_model, + secondary_site_pools=site_b_pools) + + def test_201_juju_failback(self): + """Validate controlled failback via Juju actions.""" + # Get the Ceph pools needed to failback + site_a_pools, site_b_pools = self.get_failover_pools() + + # Execute the failover Juju actions with the appropriate parameters. + # The failback operation is just a failover with sites in reverse + # order. + site_b_app_name = self.application_name + self.site_b_app_suffix + self.execute_failover_juju_actions( + primary_site_app_name=site_b_app_name, + primary_site_model=self.site_b_model, + primary_site_pools=site_b_pools, + secondary_site_app_name=self.application_name, + secondary_site_model=self.site_a_model, + secondary_site_pools=site_a_pools) + + def test_203_juju_resync(self): + """Validate the 'resync-pools' Juju action. + + The 'resync-pools' Juju action is meant to flag Ceph images from the + secondary site to re-sync against the Ceph images from the primary + site. + + This use case is useful when the Ceph secondary images are out of sync. + """ + # Get the Ceph pools needed to failback + _, site_b_pools = self.get_failover_pools() + + # Run the 'resync-pools' Juju action against the pools from site-b. + # This will make sure that the Ceph images from site-b are properly + # synced with the primary images from site-a. + site_b_app_name = self.application_name + self.site_b_app_suffix + logging.info('Re-syncing {} from model {}'.format( + site_b_app_name, self.site_b_model)) result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'resync-pools', model_name=self.site_b_model, action_params={ + 'pools': ','.join(site_b_pools), 'i-really-mean-it': True, }) logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the Ceph images from site-b report 'up+replaying' + # (which is reported by secondary Ceph images). And check that images + # exist in Cinder and Glance pools. self.wait_for_mirror_state( 'up+replaying', - application_name=self.application_name + self.site_b_app_suffix, + check_entries_behind_master=True, + application_name=site_b_app_name, model_name=self.site_b_model, - require_images_in=['cinder-ceph', 'glance']) + require_images_in=[self.cinder_ceph_app_name, 'glance'], + pools=site_b_pools) class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` destructive tests.""" - def test_kill_site_a_fail_over(self): - """Validate fail over after uncontrolled shutdown of primary.""" - for application in 'ceph-rbd-mirror', 'ceph-mon', 'ceph-osd': + def apply_cinder_ceph_workaround(self): + """Set minimal timeouts / retries to the Cinder Ceph backend. + + This is needed because the failover via Cinder API will try to do a + demotion of the site-a. However, when site-a is down, and with the + default timeouts / retries, the operation takes an unreasonably amount + of time (or sometimes it never finishes). + """ + # These new config options need to be set under the Cinder Ceph backend + # section in the main Cinder config file. + # At the moment, we don't the possibility of using Juju config to set + # these options. And also, it's not even a good practice to have them + # in production. + # These should be set only to do the Ceph failover via Cinder API, and + # they need to be removed after. + configs = { + 'rados_connect_timeout': '1', + 'rados_connection_retries': '1', + 'rados_connection_interval': '0', + 'replication_connect_timeout': '1', + } + + # Small Python script that will be executed via Juju run to update + # the Cinder config file. + update_cinder_conf_script = ( + "import configparser; " + "config = configparser.ConfigParser(); " + "config.read('/etc/cinder/cinder.conf'); " + "{}" + "f = open('/etc/cinder/cinder.conf', 'w'); " + "config.write(f); " + "f.close()") + set_cmd = '' + for cfg_name in configs: + set_cmd += "config.set('{0}', '{1}', '{2}'); ".format( + self.cinder_ceph_app_name, cfg_name, configs[cfg_name]) + script = update_cinder_conf_script.format(set_cmd) + + # Run the workaround script via Juju run + zaza.model.run_on_leader( + self.cinder_ceph_app_name, + 'python3 -c "{}"; systemctl restart cinder-volume'.format(script)) + + def kill_primary_site(self): + """Simulate an unexpected primary site shutdown.""" + logging.info('Killing the Ceph primary site') + for application in ['ceph-rbd-mirror', 'ceph-mon', 'ceph-osd']: zaza.model.remove_application( application, model_name=self.site_a_model, forcefully_remove_machines=True) + + def test_100_forced_juju_failover(self): + """Validate Ceph failover via Juju when the primary site is down. + + * Kill the primary site + * Execute the forced failover via Juju actions + """ + # Get the site-b Ceph pools that need to be promoted + _, site_b_pools = self.get_failover_pools() + site_b_app_name = self.application_name + self.site_b_app_suffix + + # Simulate primary site unexpected shutdown + self.kill_primary_site() + + # Try and promote the site-b to primary. result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'promote', model_name=self.site_b_model, action_params={ + 'pools': ','.join(site_b_pools), }) + self.assertEqual(int(result.results['Code']), 0) + + # The site-b 'promote' Juju action is expected to fail, because the + # primary site is down. self.assertEqual(result.status, 'failed') + + # Retry to promote site-b using the 'force' Juju action parameter. result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'promote', model_name=self.site_b_model, action_params={ 'force': True, + 'pools': ','.join(site_b_pools), }) + self.assertEqual(int(result.results['Code']), 0) + + # Validate successful Juju action execution self.assertEqual(result.status, 'completed') + + def test_200_forced_cinder_failover(self): + """Validate Ceph failover via Cinder when the primary site is down. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + + This assumes that the primary site is already killed. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_200_cinder_failover_without_primary_site' " + "since Cinder RBD mirroring mode is {}.".format( + cinder_rbd_mirroring_mode)) + return + + # Make sure that the Cinder Ceph backend workaround is applied. + self.apply_cinder_ceph_workaround() + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over') + + # Check that the Cinder volumes are still available after forced + # failover. + for volume in cinder.volumes.list(): + self.assertEqual(volume.status, 'available') diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 19ce1ec54..27fefbd44 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -33,7 +33,7 @@ import zaza.openstack.utilities.ceph as zaza_ceph import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.openstack.utilities.generic as zaza_utils -import zaza.openstack.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as zaza_openstack @@ -124,7 +124,7 @@ def test_ceph_osd_ceph_relation_address(self): relation_name = 'osd' remote_unit = zaza_model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( + relation = juju_utils.get_relation_from_unit( unit_name, remote_unit_name, relation_name @@ -153,7 +153,7 @@ def _ceph_to_ceph_osd_relation(self, remote_unit_name): 'ceph-public-address': remote_ip, 'fsid': fsid, } - relation = zaza_juju.get_relation_from_unit( + relation = juju_utils.get_relation_from_unit( unit_name, remote_unit_name, relation_name @@ -881,9 +881,13 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() + release_application = 'keystone' + try: + zaza_model.get_application(release_application) + except KeyError: + release_application = 'ceph-mon' cls.current_release = zaza_openstack.get_os_release( - zaza_openstack.get_current_os_release_pair( - application='ceph-mon')) + application=release_application) cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): @@ -976,7 +980,7 @@ def test_configure_compression(self): ceph_pools_detail = zaza_ceph.get_ceph_pool_details( model_name=self.model_name) logging.debug('AFTER: {}'.format(ceph_pools_detail)) - logging.debug(zaza_juju.get_relation_from_unit( + logging.debug(juju_utils.get_relation_from_unit( 'ceph-mon', self.application_name, None, model_name=self.model_name)) logging.info('Checking Ceph pool compression_mode after restoring ' diff --git a/zaza/openstack/charm_tests/charm_upgrade/tests.py b/zaza/openstack/charm_tests/charm_upgrade/tests.py index f5f301a96..f55caf042 100644 --- a/zaza/openstack/charm_tests/charm_upgrade/tests.py +++ b/zaza/openstack/charm_tests/charm_upgrade/tests.py @@ -35,6 +35,7 @@ def setUpClass(cls): """Run setup for Charm Upgrades.""" cli_utils.setup_logging() cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() cls.target_charm_namespace = '~openstack-charmers-next' def get_upgrade_url(self, charm_url): diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 7d3442e42..97b365808 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -63,7 +63,7 @@ def test_100_volume_create_extend_delete(self): self.cinder_client.volumes, vol_new.id, expected_status="available", - msg="Volume status wait") + msg="Extended volume") def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): """Create, backup, delete, restore a ceph-backed cinder volume. @@ -89,29 +89,44 @@ def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): self.assertEqual(pool_name, expected_pool) - # Create ceph-backed cinder volume - cinder_vol = self.cinder_client.volumes.create( - name='{}-410-vol'.format(self.RESOURCE_PREFIX), - size=1) - openstack_utils.resource_reaches_status( - self.cinder_client.volumes, - cinder_vol.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='Volume status wait') + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3)): + with attempt: + # Create ceph-backed cinder volume + cinder_vol_name = '{}-410-{}-vol'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) + cinder_vol = self.cinder_client.volumes.create( + name=cinder_vol_name, size=1) + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='ceph-backed cinder volume') + + # Back up the volume + # NOTE(lourot): sometimes, especially on Mitaka, the backup + # remains stuck forever in 'creating' state and the volume in + # 'backing-up' state. See lp:1877076 + # Attempting to create another volume and another backup + # usually then succeeds. Release notes and bug trackers show + # that many things have been fixed and are still left to be + # fixed in this area. + # When the backup creation succeeds, it usually does within + # 12 minutes. + vol_backup_name = '{}-410-{}-backup-vol'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) + vol_backup = self.cinder_client.backups.create( + cinder_vol.id, name=vol_backup_name) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Backup volume') - # Backup the volume - vol_backup = self.cinder_client.backups.create( - cinder_vol.id, - name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX)) - openstack_utils.resource_reaches_status( - self.cinder_client.backups, - vol_backup.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='Volume status wait') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -122,7 +137,7 @@ def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): wait_iteration_max_time=180, stop_after_attempt=15, expected_status='available', - msg='Backup status wait') + msg='Restored backup volume') # Delete the backup openstack_utils.delete_volume_backup( self.cinder_client, @@ -143,12 +158,12 @@ def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): obj_count_samples.append(obj_count) pool_size_samples.append(kb_used) - name = '{}-410-vol'.format(self.RESOURCE_PREFIX) vols = self.cinder_client.volumes.list() try: - cinder_vols = [v for v in vols if v.name == name] + cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == name] + cinder_vols = [v for v in vols if + v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index c01014e6c..5e701413c 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -23,9 +23,9 @@ import designateclient.v1.servers as servers import zaza.model -import zaza.openstack.utilities.juju as zaza_juju -import zaza.openstack.utilities.generic as generic_utils +import zaza.utilities.juju as juju_utils import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.designate.utils as designate_utils import zaza.charm_lifecycle.utils as lifecycle_utils @@ -179,7 +179,7 @@ def wait(): reraise=True ) def _wait_to_resolve_test_record(self): - dns_ip = zaza_juju.get_relation_from_unit( + dns_ip = juju_utils.get_relation_from_unit( 'designate/0', 'designate-bind/0', 'dns-backend' diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index ab0a3b31d..367c980c5 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -14,7 +14,11 @@ """Code for configuring glance.""" +import json import logging + +import boto3 +import zaza.model as model import zaza.openstack.utilities.openstack as openstack_utils import zaza.utilities.deployment_env as deployment_env @@ -32,8 +36,37 @@ def basic_setup(): """ +def _get_default_glance_client(): + """Create default Glance client using overcloud credentials.""" + keystone_session = openstack_utils.get_overcloud_keystone_session() + glance_client = openstack_utils.get_glance_session_client(keystone_session) + return glance_client + + +def get_stores_info(glance_client=None): + """Retrieve glance backing store info. + + :param glance_client: Authenticated glanceclient + :type glance_client: glanceclient.Client + """ + glance_client = glance_client or _get_default_glance_client() + stores = glance_client.images.get_stores_info().get("stores", []) + return stores + + +def get_store_ids(glance_client=None): + """Retrieve glance backing store ids. + + :param glance_client: Authenticated glanceclient + :type glance_client: glanceclient.Client + """ + stores = get_stores_info(glance_client) + return [store["id"] for store in stores] + + def add_image(image_url, glance_client=None, image_name=None, tags=[], - properties=None): + properties=None, backend=None, disk_format='qcow2', + visibility='public', container_format='bare'): """Retrieve image from ``image_url`` and add it to glance. :param image_url: Retrievable URL with image data @@ -47,10 +80,14 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[], :param properties: Properties to add to image :type properties: dict """ - if not glance_client: - keystone_session = openstack_utils.get_overcloud_keystone_session() - glance_client = openstack_utils.get_glance_session_client( - keystone_session) + glance_client = glance_client or _get_default_glance_client() + if backend is not None: + stores = get_store_ids(glance_client) + if backend not in stores: + raise ValueError("Invalid backend: %(backend)s " + "(available: %(available)s)" % { + "backend": backend, + "available": ", ".join(stores)}) if image_name: image = openstack_utils.get_images_by_name( glance_client, image_name) @@ -65,7 +102,11 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[], image_url, image_name, tags=tags, - properties=properties) + properties=properties, + backend=backend, + disk_format=disk_format, + visibility=visibility, + container_format=container_format) def add_cirros_image(glance_client=None, image_name=None): @@ -124,3 +165,58 @@ def add_lts_image(glance_client=None, image_name=None, release=None, glance_client=glance_client, image_name=image_name, properties=properties) + + +def configure_external_s3_backend(): + """Set up Ceph-radosgw as an external S3 backend for Glance.""" + logging.info("Creating a test S3 user and credentials for Glance") + username, displayname = "zaza-glance-test", "Zaza Glance Test User" + cmd = "radosgw-admin user create --uid='{}' --display-name='{}'".format( + username, displayname + ) + results = model.run_on_leader("ceph-mon", cmd) + stdout = json.loads(results["stdout"]) + keys = stdout["keys"][0] + access_key, secret_key = keys["access_key"], keys["secret_key"] + + logging.info("Getting S3 endpoint URL of Radosgw from Keystone") + keystone_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client(keystone_auth) + endpoint_url = keystone_client.session.get_endpoint( + service_type="s3", + interface="public", + region="RegionOne", + ) + + logging.info("Creating a test S3 bucket for Glance") + bucket_name = "zaza-glance-s3-test" + s3_client = boto3.client( + "s3", + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + ) + s3_client.create_bucket(Bucket=bucket_name) + + logging.info("Updating Glance configs with S3 endpoint information") + model.set_application_config( + "glance", + { + "s3-store-host": endpoint_url, + "s3-store-access-key": access_key, + "s3-store-secret-key": secret_key, + "s3-store-bucket": bucket_name, + }, + ) + model.wait_for_agent_status() + + logging.info("Waiting for units to reach target states") + model.wait_for_application_states( + states={ + "glance": { + "workload-status": "active", + "workload-status-message": "Unit is ready", + } + } + ) + model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 363032d24..a6da2912b 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -18,8 +18,10 @@ import logging -import zaza.openstack.utilities.openstack as openstack_utils +import boto3 +import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils class GlanceTest(test_utils.OpenStackBaseTest): @@ -67,6 +69,32 @@ def test_411_set_disk_format(self): {'image_format': {'disk_formats': ['qcow2']}}, ['glance-api']) + def test_412_image_conversion(self): + """Check image-conversion config. + + When image-conversion config is enabled glance will convert images + to raw format, this is only performed for interoperable image import + docs.openstack.org/glance/train/admin/interoperable-image-import.html + image conversion is done at server-side for better image handling + """ + current_release = openstack_utils.get_os_release() + bionic_stein = openstack_utils.get_os_release('bionic_stein') + if current_release < bionic_stein: + self.skipTest('image-conversion config is supported since ' + 'bionic_stein or newer versions') + + with self.config_change({'image-conversion': 'false'}, + {'image-conversion': 'true'}): + image_url = openstack_utils.find_cirros_image(arch='x86_64') + image = openstack_utils.create_image( + self.glance_client, + image_url, + 'cirros-test-import', + force_import=True) + + disk_format = self.glance_client.images.get(image.id).disk_format + self.assertEqual('raw', disk_format) + def test_900_restart_on_config_change(self): """Checking restart happens on config change.""" # Config file affected by juju set config change @@ -92,3 +120,105 @@ def test_901_pause_resume(self): they are started """ self.pause_resume(['glance-api']) + + +class GlanceCephRGWBackendTest(test_utils.OpenStackBaseTest): + """Encapsulate glance tests using the Ceph RGW backend. + + It validates the Ceph RGW backend in glance, which uses the Swift API. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests.""" + super(GlanceCephRGWBackendTest, cls).setUpClass() + + swift_session = openstack_utils.get_keystone_session_from_relation( + 'ceph-radosgw') + cls.swift = openstack_utils.get_swift_session_client( + swift_session) + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session) + + def test_100_create_image(self): + """Create an image and do a simple validation of it. + + The OpenStack Swift API is used to do the validation, since the Ceph + Rados Gateway serves an API which is compatible with that. + """ + image_name = 'zaza-ceph-rgw-image' + openstack_utils.create_image( + glance=self.glance_client, + image_url=openstack_utils.find_cirros_image(arch='x86_64'), + image_name=image_name, + backend='swift') + headers, containers = self.swift.get_account() + self.assertEqual(len(containers), 1) + container_name = containers[0].get('name') + headers, objects = self.swift.get_container(container_name) + images = openstack_utils.get_images_by_name( + self.glance_client, + image_name) + self.assertEqual(len(images), 1) + image = images[0] + total_bytes = 0 + for ob in objects: + if '{}-'.format(image['id']) in ob['name']: + total_bytes = total_bytes + int(ob['bytes']) + logging.info( + 'Checking glance image size {} matches swift ' + 'image size {}'.format(image['size'], total_bytes)) + self.assertEqual(image['size'], total_bytes) + openstack_utils.delete_image(self.glance_client, image['id']) + + +class GlanceExternalS3Test(test_utils.OpenStackBaseTest): + """Encapsulate glance tests using an external S3 backend.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests with S3 backend.""" + super(GlanceExternalS3Test, cls).setUpClass() + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session + ) + + configs = model.get_application_config("glance") + cls.s3_store_host = configs["s3-store-host"]["value"] + cls.s3_store_access_key = configs["s3-store-access-key"]["value"] + cls.s3_store_secret_key = configs["s3-store-secret-key"]["value"] + cls.s3_store_bucket = configs["s3-store-bucket"]["value"] + + def test_100_create_delete_image(self): + """Create an image and do a simple validation of it. + + Validate the size of the image in both Glance API and actual S3 bucket. + """ + image_name = "zaza-s3-test-image" + openstack_utils.create_image( + glance=self.glance_client, + image_url=openstack_utils.find_cirros_image(arch="x86_64"), + image_name=image_name, + backend="s3", + ) + images = openstack_utils.get_images_by_name( + self.glance_client, image_name + ) + self.assertEqual(len(images), 1) + image = images[0] + + s3_client = boto3.client( + "s3", + endpoint_url=self.s3_store_host, + aws_access_key_id=self.s3_store_access_key, + aws_secret_access_key=self.s3_store_secret_key, + ) + response = s3_client.head_object( + Bucket=self.s3_store_bucket, Key=image["id"] + ) + logging.info( + "Checking glance image size {} matches S3 object's ContentLength " + "{}".format(image["size"], response["ContentLength"]) + ) + self.assertEqual(image["size"], response["ContentLength"]) + openstack_utils.delete_image(self.glance_client, image["id"]) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py index 06e172d1d..96648af2f 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py @@ -17,9 +17,32 @@ """Code for configuring glance-simplestreams-sync.""" import logging +import tenacity +import pprint import zaza.model as zaza_model import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def _get_catalog(): + """Retrieve the Keystone service catalog. + + :returns: The raw Keystone service catalog. + :rtype: List[Dict] + """ + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + + token = keystone_session.get_token() + token_data = keystone_client.tokens.get_token_data(token) + + if 'catalog' not in token_data['token']: + raise ValueError('catalog not in token data: "{}"' + .format(pprint.pformat(token_data))) + + return token_data['token']['catalog'] def sync_images(): @@ -30,11 +53,27 @@ def sync_images(): deployment. """ logging.info("Synchronising images using glance-simplestreams-sync") - generic_utils.assertActionRanOK( - zaza_model.run_action_on_leader( - "glance-simplestreams-sync", - "sync-images", - raise_on_failure=True, - action_params={}, - ) - ) + + catalog = None + try: + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10), + reraise=True): + with attempt: + # Proactively retrieve the Keystone service catalog so that we + # can log it in the event of a failure. + catalog = _get_catalog() + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "glance-simplestreams-sync", + "sync-images", + raise_on_failure=True, + action_params={}, + ) + ) + except Exception: + logging.info('Contents of Keystone service catalog: "{}"' + .format(pprint.pformat(catalog))) + raise diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index af47b76ff..941414590 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -76,57 +76,121 @@ def test_920_put_in_maintenance(self): self._toggle_maintenance_and_wait('false') -class HaclusterScalebackTest(HaclusterBaseTest): - """hacluster scaleback tests.""" +class HaclusterScaleBackAndForthTest(HaclusterBaseTest): + """hacluster tests scaling back and forth.""" @classmethod def setUpClass(cls): - """Run class setup for running hacluster scaleback tests.""" - super(HaclusterScalebackTest, cls).setUpClass() + """Run class setup for running hacluster tests.""" + super(HaclusterScaleBackAndForthTest, cls).setUpClass() test_config = cls.test_config['tests_options']['hacluster'] cls._principle_app_name = test_config['principle-app-name'] cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): - """Remove a unit and add a new one.""" + """Remove one unit, recalculate quorum and re-add one unit. + + NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't + recalculated when removing units. So within a cluster of 3 units, + removing a unit and re-adding one led to a situation where corosync + considers having 3 nodes online out of 4, instead of just 3 out of 3. + This test covers this scenario. + """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) - doomed_principle_unit = principle_units[0] - other_principle_unit = principle_units[1] + surviving_principle_unit = principle_units[0] + doomed_principle_unit = principle_units[1] + surviving_hacluster_unit = juju_utils.get_subordinate_units( + [surviving_principle_unit], + charm_name=self._hacluster_charm_name)[0] doomed_hacluster_unit = juju_utils.get_subordinate_units( - [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] - other_hacluster_unit = juju_utils.get_subordinate_units( - [other_principle_unit], charm_name=self._hacluster_charm_name)[0] + [doomed_principle_unit], + charm_name=self._hacluster_charm_name)[0] logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) zaza.model.run_action( doomed_hacluster_unit, 'pause', raise_on_failure=True) - logging.info('OK') logging.info('Removing {}'.format(doomed_principle_unit)) zaza.model.destroy_unit( self._principle_app_name, doomed_principle_unit, wait_disappear=True) - logging.info('OK') logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, + 'blocked') + # NOTE(lourot): the surviving principle units (usually keystone units) + # aren't guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() - logging.info('OK') - logging.info('Adding an hacluster unit') + # At this point the corosync ring hasn't been updated yet, so it should + # still remember the deleted unit: + self.__assert_some_corosync_nodes_are_offline(surviving_hacluster_unit) + + logging.info('Updating corosync ring') + hacluster_app_name = zaza.model.get_unit_from_name( + surviving_hacluster_unit).application + zaza.model.run_action_on_leader( + hacluster_app_name, + 'update-ring', + action_params={'i-really-mean-it': True}, + raise_on_failure=True) + + # At this point if the corosync ring has been properly updated, there + # shouldn't be any trace of the deleted unit anymore: + self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) + + logging.info('Re-adding an hacluster unit') zaza.model.add_unit(self._principle_app_name, wait_appear=True) - logging.info('OK') logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') - # NOTE(lourot): the principle application remains blocked after scaling - # back up until lp:1400481 is solved. - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + # NOTE(lourot): the principle charm may remain blocked here. This seems + # to happen often when it is keystone and has a mysql-router as other + # subordinate charm. The keystone units seems to often remain blocked + # with 'Database not initialised'. This is not the hacluster charm's + # fault and this is why we don't validate here that the entire model + # goes back to active/idle. + zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, + 'active') zaza.model.block_until_all_units_idle() - logging.debug('OK') + + # Because of lp:1874719 the corosync ring may show a mysterious offline + # 'node1' node. We clean up the ring by re-running the 'update-ring' + # action: + logging.info('Updating corosync ring - workaround for lp:1874719') + zaza.model.run_action_on_leader( + hacluster_app_name, + 'update-ring', + action_params={'i-really-mean-it': True}, + raise_on_failure=True) + + # At this point the corosync ring should not contain any offline node: + self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) + + def __assert_some_corosync_nodes_are_offline(self, hacluster_unit): + logging.info('Checking that corosync considers at least one node to ' + 'be offline') + output = self._get_crm_status(hacluster_unit) + self.assertIn('OFFLINE', output, + "corosync should list at least one offline node") + + def __assert_all_corosync_nodes_are_online(self, hacluster_unit): + logging.info('Checking that corosync considers all nodes to be online') + output = self._get_crm_status(hacluster_unit) + self.assertNotIn('OFFLINE', output, + "corosync shouldn't list any offline node") + + @staticmethod + def _get_crm_status(hacluster_unit): + cmd = 'sudo crm status' + result = zaza.model.run_on_unit(hacluster_unit, cmd) + code = result.get('Code') + if code != '0': + raise zaza.model.CommandRunFailed(cmd, result) + output = result.get('Stdout').strip() + logging.debug('crm output received: {}'.format(output)) + return output diff --git a/zaza/openstack/charm_tests/ironic/__init__.py b/zaza/openstack/charm_tests/ironic/__init__.py new file mode 100644 index 000000000..aedad0501 --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ironic.""" diff --git a/zaza/openstack/charm_tests/ironic/setup.py b/zaza/openstack/charm_tests/ironic/setup.py new file mode 100644 index 000000000..fe7d0345c --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/setup.py @@ -0,0 +1,184 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring ironic.""" + +import copy +import os +import tenacity + +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack.utilities import ( + cli as cli_utils, +) +import zaza.model as zaza_model + + +FLAVORS = { + 'bm1.small': { + 'flavorid': 2, + 'ram': 2048, + 'disk': 20, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL1_SMALL": 1, + }, + }, + 'bm1.medium': { + 'flavorid': 3, + 'ram': 4096, + 'disk': 40, + 'vcpus': 2, + 'properties': { + "resources:CUSTOM_BAREMETAL1_MEDIUM": 1, + }, + }, + 'bm1.large': { + 'flavorid': 4, + 'ram': 8192, + 'disk': 40, + 'vcpus': 4, + 'properties': { + "resources:CUSTOM_BAREMETAL1_LARGE": 1, + }, + }, + 'bm1.tempest': { + 'flavorid': 6, + 'ram': 256, + 'disk': 1, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL1_TEMPEST": 1, + }, + }, + 'bm2.tempest': { + 'flavorid': 7, + 'ram': 512, + 'disk': 1, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL2_TEMPEST": 1, + }, + }, +} + + +def _add_image(url, image_name, backend="swift", + disk_format="raw", container_format="bare"): + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + reraise=True): + with attempt: + glance_setup.add_image( + url, + image_name=image_name, + backend=backend, + disk_format=disk_format, + container_format=container_format) + + +def add_ironic_deployment_image(initrd_url=None, kernel_url=None): + """Add Ironic deploy images to glance. + + :param initrd_url: URL where the ari image resides + :type initrd_url: str + :param kernel_url: URL where the aki image resides + :type kernel_url: str + """ + base_name = 'ironic-deploy' + initrd_name = "{}-initrd".format(base_name) + vmlinuz_name = "{}-vmlinuz".format(base_name) + if not initrd_url: + initrd_url = os.environ.get('TEST_IRONIC_DEPLOY_INITRD', None) + if not kernel_url: + kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None) + if not all([initrd_url, kernel_url]): + raise ValueError("Missing required deployment image URLs") + + _add_image( + initrd_url, + initrd_name, + backend="swift", + disk_format="ari", + container_format="ari") + + _add_image( + kernel_url, + vmlinuz_name, + backend="swift", + disk_format="aki", + container_format="aki") + + +def add_ironic_os_image(image_url=None): + """Upload the operating system images built for bare metal deployments. + + :param image_url: URL where the image resides + :type image_url: str + """ + image_url = image_url or os.environ.get( + 'TEST_IRONIC_RAW_BM_IMAGE', None) + image_name = "baremetal-ubuntu-image" + if image_url is None: + raise ValueError("Missing image_url") + + _add_image( + image_url, + image_name, + backend="swift", + disk_format="raw", + container_format="bare") + + +def set_temp_url_secret(): + """Run the set-temp-url-secret on the ironic-conductor leader. + + This is needed if direct boot method is enabled. + """ + zaza_model.run_action_on_leader( + 'ironic-conductor', + 'set-temp-url-secret', + action_params={}) + + +def create_bm_flavors(nova_client=None): + """Create baremetal flavors. + + :param nova_client: Authenticated nova client + :type nova_client: novaclient.v2.client.Client + """ + if not nova_client: + keystone_session = openstack_utils.get_overcloud_keystone_session() + nova_client = openstack_utils.get_nova_session_client( + keystone_session) + cli_utils.setup_logging() + names = [flavor.name for flavor in nova_client.flavors.list()] + # Disable scheduling based on standard flavor properties + default_properties = { + "resources:VCPU": 0, + "resources:MEMORY_MB": 0, + "resources:DISK_GB": 0, + } + for flavor in FLAVORS.keys(): + if flavor not in names: + properties = copy.deepcopy(default_properties) + properties.update(FLAVORS[flavor]["properties"]) + bm_flavor = nova_client.flavors.create( + name=flavor, + ram=FLAVORS[flavor]['ram'], + vcpus=FLAVORS[flavor]['vcpus'], + disk=FLAVORS[flavor]['disk'], + flavorid=FLAVORS[flavor]['flavorid']) + bm_flavor.set_keys(properties) diff --git a/zaza/openstack/charm_tests/ironic/tests.py b/zaza/openstack/charm_tests/ironic/tests.py new file mode 100644 index 000000000..9cfb85a9b --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/tests.py @@ -0,0 +1,83 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate ironic testing.""" + +import logging + +import ironicclient.client as ironic_client +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def _get_ironic_client(ironic_api_version="1.58"): + keystone_session = openstack_utils.get_overcloud_keystone_session() + ironic = ironic_client.Client(1, session=keystone_session, + os_ironic_api_version=ironic_api_version) + return ironic + + +class IronicTest(test_utils.OpenStackBaseTest): + """Run Ironic specific tests.""" + + _SERVICES = ['ironic-api'] + + def test_110_catalog_endpoints(self): + """Verify that the endpoints are present in the catalog.""" + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + actual_endpoints = keystone_client.service_catalog.get_endpoints() + actual_interfaces = [endpoint['interface'] for endpoint in + actual_endpoints["baremetal"]] + for expected_interface in ('internal', 'admin', 'public'): + assert(expected_interface in actual_interfaces) + + def test_400_api_connection(self): + """Simple api calls to check service is up and responding.""" + ironic = _get_ironic_client() + + logging.info('listing conductors') + conductors = ironic.conductor.list() + assert(len(conductors) > 0) + + # By default, only IPMI HW type is enabled. iDrac and Redfish + # can optionally be enabled + drivers = ironic.driver.list() + driver_names = [drv.name for drv in drivers] + + expected = ['intel-ipmi', 'ipmi'] + for exp in expected: + assert(exp in driver_names) + assert(len(driver_names) == 2) + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + self.restart_on_changed_debug_oslo_config_file( + '/etc/ironic/ironic.conf', self._SERVICES) + + def test_910_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + logging.info('Skipping pause resume test LP: #1886202...') + return + with self.pause_resume(self._SERVICES): + logging.info("Testing pause resume") diff --git a/zaza/openstack/charm_tests/kerberos/setup.py b/zaza/openstack/charm_tests/kerberos/setup.py index d441f8c11..11f4afa80 100644 --- a/zaza/openstack/charm_tests/kerberos/setup.py +++ b/zaza/openstack/charm_tests/kerberos/setup.py @@ -113,7 +113,15 @@ def retrieve_and_attach_keytab(): 'keystone_keytab', tmp_file) - zaza.model.wait_for_application_states() + # cs:ubuntu charm has changed behaviour and we can't rely on the workload + # staus message. Thus, ignore it. + states = { + "ubuntu-test-host": { + "workload-status": "active", + "workload-status-message": "", + } + } + zaza.model.wait_for_application_states(states=states) zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 748439f19..73264cd7b 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -41,9 +41,8 @@ def wait_for_cacert(model_name=None): :type model_name: str """ logging.info("Waiting for cacert") - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', - openstack_utils.KEYSTONE_REMOTE_CACERT, 'CERTIFICATE', model_name=model_name) zaza.model.block_until_all_units_idle(model_name=model_name) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index ed5f42a09..477152d0e 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -229,7 +229,7 @@ def test_end_user_domain_admin_access(self): 'OS_DOMAIN_NAME': DEMO_DOMAIN, } if self.tls_rid: - openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) @@ -259,7 +259,7 @@ def test_end_user_access_and_token(self): """ def _validate_token_data(openrc): if self.tls_rid: - openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) @@ -380,7 +380,12 @@ def test_security_checklist(self): class LdapTests(BaseKeystoneTest): - """Keystone ldap tests tests.""" + """Keystone ldap tests.""" + + non_string_type_keys = ('ldap-user-enabled-mask', + 'ldap-user-enabled-invert', + 'ldap-group-members-are-ids', + 'ldap-use-pool') @classmethod def setUpClass(cls): @@ -402,13 +407,22 @@ def _get_ldap_config(self): 'ldap-password': 'crapper', 'ldap-suffix': 'dc=test,dc=com', 'domain-name': 'userdomain', + 'ldap-config-flags': + { + 'group_tree_dn': 'ou=groups,dc=test,dc=com', + 'group_objectclass': 'posixGroup', + 'group_name_attribute': 'cn', + 'group_member_attribute': 'memberUid', + 'group_members_are_ids': 'true', + } } - def _find_keystone_v3_user(self, username, domain): + def _find_keystone_v3_user(self, username, domain, group=None): """Find a user within a specified keystone v3 domain. :param str username: Username to search for in keystone :param str domain: username selected from which domain + :param str group: group to search for in keystone for group membership :return: return username if found :rtype: Optional[str] """ @@ -418,9 +432,15 @@ def _find_keystone_v3_user(self, username, domain): openstack_utils.get_overcloud_auth(address=ip)) client = openstack_utils.get_keystone_session_client(session) - domain_users = client.users.list( - domain=client.domains.find(name=domain).id - ) + if group is None: + domain_users = client.users.list( + domain=client.domains.find(name=domain).id, + ) + else: + domain_users = client.users.list( + domain=client.domains.find(name=domain).id, + group=self._find_keystone_v3_group(group, domain).id, + ) usernames = [u.name.lower() for u in domain_users] if username.lower() in usernames: @@ -431,27 +451,232 @@ def _find_keystone_v3_user(self, username, domain): ) return None + def _find_keystone_v3_group(self, group, domain): + """Find a group within a specified keystone v3 domain. + + :param str group: Group to search for in keystone + :param str domain: group selected from which domain + :return: return group if found + :rtype: Optional[str] + """ + for ip in self.keystone_ips: + logging.info('Keystone IP {}'.format(ip)) + session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + client = openstack_utils.get_keystone_session_client(session) + + domain_groups = client.groups.list( + domain=client.domains.find(name=domain).id + ) + + for searched_group in domain_groups: + if searched_group.name.lower() == group.lower(): + return searched_group + + logging.debug( + "Group {} was not found. Returning None.".format(group) + ) + return None + def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" application_name = 'keystone-ldap' - config = self._get_ldap_config() + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) with self.config_change( - self.config_current(application_name, config.keys()), - config, - application_name=application_name): - logging.info( - 'Waiting for users to become available in keystone...' - ) - test_config = lifecycle_utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get("target_deploy_status", {}) - ) + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for users to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(jamespage): Test fixture should have + # johndoe and janedoe accounts + johndoe = self._find_keystone_v3_user( + 'john doe', 'userdomain') + self.assertIsNotNone( + johndoe, "user 'john doe' was unknown") + janedoe = self._find_keystone_v3_user( + 'jane doe', 'userdomain') + self.assertIsNotNone( + janedoe, "user 'jane doe' was unknown") + + def test_101_keystone_ldap_groups(self): + """Validate basic functionality of keystone API with ldap.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) - with self.v3_keystone_preferred(): - # NOTE(jamespage): Test fixture should have johndoe and janedoe - # accounts - johndoe = self._find_keystone_v3_user('john doe', 'userdomain') - self.assertIsNotNone(johndoe, "user 'john doe' was unknown") - janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') - self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for groups to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(arif-ali): Test fixture should have openstack and + # admin groups + openstack_group = self._find_keystone_v3_group( + 'openstack', 'userdomain') + self.assertIsNotNone( + openstack_group.name, "group 'openstack' was unknown") + admin_group = self._find_keystone_v3_group( + 'admin', 'userdomain') + self.assertIsNotNone( + admin_group.name, "group 'admin' was unknown") + + def test_102_keystone_ldap_group_membership(self): + """Validate basic functionality of keystone API with ldap.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for groups to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(arif-ali): Test fixture should have openstack and + # admin groups + openstack_group = self._find_keystone_v3_user( + 'john doe', 'userdomain', group='openstack') + self.assertIsNotNone( + openstack_group, + "john doe was not in group 'openstack'") + admin_group = self._find_keystone_v3_user( + 'john doe', 'userdomain', group='admin') + self.assertIsNotNone( + admin_group, "'john doe' was not in group 'admin'") + + +class LdapExplicitCharmConfigTests(LdapTests): + """Keystone ldap tests.""" + + def _get_ldap_config(self): + """Generate ldap config for current model. + + :return: tuple of whether ldap-server is running and if so, config + for the keystone-ldap application. + :rtype: Tuple[bool, Dict[str,str]] + """ + ldap_ips = zaza.model.get_app_ips("ldap-server") + self.assertTrue(ldap_ips, "Should be at least one ldap server") + return { + 'ldap-server': "ldap://{}".format(ldap_ips[0]), + 'ldap-user': 'cn=admin,dc=test,dc=com', + 'ldap-password': 'crapper', + 'ldap-suffix': 'dc=test,dc=com', + 'domain-name': 'userdomain', + 'ldap-query-scope': 'one', + 'ldap-user-objectclass': 'inetOrgPerson', + 'ldap-user-id-attribute': 'cn', + 'ldap-user-name-attribute': 'sn', + 'ldap-user-enabled-attribute': 'enabled', + 'ldap-user-enabled-invert': False, + 'ldap-user-enabled-mask': 0, + 'ldap-user-enabled-default': 'True', + 'ldap-group-tree-dn': 'ou=groups,dc=test,dc=com', + 'ldap-group-objectclass': '', + 'ldap-group-id-attribute': 'cn', + 'ldap-group-name-attribute': 'cn', + 'ldap-group-member-attribute': 'memberUid', + 'ldap-group-members-are-ids': True, + 'ldap-config-flags': '{group_objectclass: "posixGroup",' + ' use_pool: True,' + ' group_tree_dn: "group_tree_dn_foobar"}', + } + + def test_200_config_flags_precedence(self): + """Validates precedence when the same config options are used.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Performing LDAP settings validation in keystone.conf...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + units = zaza.model.get_units("keystone-ldap", + model_name=self.model_name) + result = zaza.model.run_on_unit( + units[0].name, + "cat /etc/keystone/domains/keystone.userdomain.conf") + # not present in charm config, but present in config flags + self.assertIn("use_pool = True", result['stdout'], + "use_pool value is expected to be present and " + "set to True in the config file") + # ldap-config-flags overriding empty charm config value + self.assertIn("group_objectclass = posixGroup", + result['stdout'], + "group_objectclass is expected to be present and" + " set to posixGroup in the config file") + # overridden by charm config, not written to file + self.assertNotIn( + "group_tree_dn_foobar", + result['stdout'], + "user_tree_dn ldap-config-flags value needs to be " + "overridden by ldap-user-tree-dn in config file") + # complementing the above, value used is from charm setting + self.assertIn("group_tree_dn = ou=groups", result['stdout'], + "user_tree_dn value is expected to be present " + "and set to dc=test,dc=com in the config file") diff --git a/zaza/openstack/charm_tests/magpie/__init__.py b/zaza/openstack/charm_tests/magpie/__init__.py new file mode 100644 index 000000000..7fd0805b1 --- /dev/null +++ b/zaza/openstack/charm_tests/magpie/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing Magpie.""" diff --git a/zaza/openstack/charm_tests/magpie/tests.py b/zaza/openstack/charm_tests/magpie/tests.py new file mode 100644 index 000000000..0ac25a64e --- /dev/null +++ b/zaza/openstack/charm_tests/magpie/tests.py @@ -0,0 +1,82 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Magpie testing.""" + +import logging + +import zaza + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils + + +class MagpieTest(test_utils.BaseCharmTest): + """Base Magpie tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for Magpie charm operation tests.""" + super(MagpieTest, cls).setUpClass() + unit_names = sorted( + [i.entity_id + for i in zaza.model.get_units('magpie')]) + cls.test_unit_0 = unit_names[0] + cls.test_unit_1 = unit_names[1] + + def test_break_dns_single(self): + """Check DNS failure is reflected in workload status.""" + zaza.model.run_on_unit( + self.test_unit_0, + 'mv /etc/resolv.conf /etc/resolv.conf.bak') + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*rev dns failed.*') + logging.info('Restoring /etc/resolv.conf') + zaza.model.run_on_unit( + self.test_unit_0, + 'mv /etc/resolv.conf.bak /etc/resolv.conf') + logging.info('Updating status') + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + + def test_break_ping_single(self): + """Check ping failure is reflected in workload status.""" + icmp = "iptables {} INPUT -p icmp --icmp-type echo-request -j REJECT" + logging.info('Blocking ping on {}'.format(self.test_unit_1)) + zaza.model.run_on_unit( + self.test_unit_1, + icmp.format('--append')) + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + logging.info('Checking status on {}'.format(self.test_unit_0)) + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*icmp failed.*') + logging.info('Allowing ping on {}'.format(self.test_unit_1)) + zaza.model.run_on_unit( + self.test_unit_1, + icmp.format('--delete')) + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + logging.info('Checking status on {}'.format(self.test_unit_0)) + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*icmp ok.*') diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 8a1fdeec2..1b58f658d 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -16,11 +16,47 @@ """Encapsulate Manila testing.""" +import logging import tenacity from manilaclient import client as manilaclient +import zaza.model +import zaza.openstack.configure.guest as guest +import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.charm_tests.neutron.tests as neutron_tests + + +def verify_status(stdin, stdout, stderr): + """Callable to verify the command output. + + It checks if the command successfully executed. + + This is meant to be given as parameter 'verify' to the helper function + 'openstack_utils.ssh_command'. + """ + status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) + assert status == 0 + + +def verify_manila_testing_file(stdin, stdout, stderr): + """Callable to verify the command output. + + It checks if the command successfully executed, and it validates the + testing file written on the Manila share. + + This is meant to be given as parameter 'verify' to the helper function + 'openstack_utils.ssh_command'. + """ + verify_status(stdin, stdout, stderr) + out = "" + for line in iter(stdout.readline, ""): + out += line + assert out == "test\n" class ManilaTests(test_utils.OpenStackBaseTest): @@ -35,6 +71,12 @@ def setUpClass(cls): def test_manila_api(self): """Test that the Manila API is working.""" + # The manila charm contains a 'band-aid' for Bug #1706699 which relies + # on update-status to bring up services if needed. When the tests run + # an update-status hook might not have run so services may still be + # stopped so force a hook execution. + for unit in zaza.model.get_units('manila'): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") self.assertEqual([], self._list_shares()) @tenacity.retry( @@ -42,3 +84,232 @@ def test_manila_api(self): wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) def _list_shares(self): return self.manila_client.shares.list() + + +class ManilaBaseTest(test_utils.OpenStackBaseTest): + """Encapsulate a Manila basic functionality test.""" + + RESOURCE_PREFIX = 'zaza-manilatests' + INSTANCE_KEY = 'bionic' + INSTANCE_USERDATA = """#cloud-config +packages: +- nfs-common +""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaBaseTest, cls).setUpClass() + cls.nova_client = openstack_utils.get_nova_session_client( + session=cls.keystone_session) + cls.manila_client = manilaclient.Client( + session=cls.keystone_session, client_version='2') + cls.share_name = 'test-manila-share' + cls.share_type_name = 'default_share_type' + cls.share_protocol = 'nfs' + cls.mount_dir = '/mnt/manila_share' + cls.share_network = None + + @classmethod + def tearDownClass(cls): + """Run class teardown after tests finished.""" + # Cleanup Nova servers + logging.info('Cleaning up test Nova servers') + fips_reservations = [] + for vm in cls.nova_client.servers.list(): + fips_reservations += neutron_tests.floating_ips_from_instance(vm) + vm.delete() + openstack_utils.resource_removed( + cls.nova_client.servers, + vm.id, + msg="Waiting for the Nova VM {} to be deleted".format(vm.name)) + + # Delete FiPs reservations + logging.info('Cleaning up test FiPs reservations') + neutron = openstack_utils.get_neutron_session_client( + session=cls.keystone_session) + for fip in neutron.list_floatingips()['floatingips']: + if fip['floating_ip_address'] in fips_reservations: + neutron.delete_floatingip(fip['id']) + + # Cleanup Manila shares + logging.info('Cleaning up test shares') + for share in cls.manila_client.shares.list(): + share.delete() + openstack_utils.resource_removed( + cls.manila_client.shares, + share.id, + msg="Waiting for the Manila share {} to be deleted".format( + share.name)) + + # Cleanup test Manila share servers (spawned by the driver when DHSS + # is enabled). + logging.info('Cleaning up test shares servers (if found)') + for server in cls.manila_client.share_servers.list(): + server.delete() + openstack_utils.resource_removed( + cls.manila_client.share_servers, + server.id, + msg="Waiting for the share server {} to be deleted".format( + server.id)) + + def _get_mount_options(self): + """Get the appropriate mount options used to mount the Manila share. + + :returns: The proper mount options flags for the share protocol. + :rtype: string + """ + if self.share_protocol == 'nfs': + return 'nfsvers=4.1,proto=tcp' + else: + raise NotImplementedError( + 'Share protocol not supported yet: {}'.format( + self.share_protocol)) + + def _mount_share_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key, share_path): + """Mount a share into a Nova instance. + + The mount command is executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + :param share_path: Share network path. + :type share_path: string + """ + ssh_cmd = ( + 'sudo mkdir -p {0} && ' + 'sudo mount -t {1} -o {2} {3} {0}'.format( + self.mount_dir, + self.share_protocol, + self._get_mount_options(), + share_path)) + + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)): + with attempt: + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command=ssh_cmd, + verify=verify_status) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _write_testing_file_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Write a file on a Manila share mounted into a Nova instance. + + Write a testing file into the already mounted Manila share from the + given Nova instance (which is meant to be validated from another + instance). These commands are executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='echo "test" | sudo tee {}/test'.format( + self.mount_dir), + verify=verify_status) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _validate_testing_file_from_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Validate a file from the Manila share mounted into a Nova instance. + + This is meant to run after the testing file was already written into + another Nova instance. It validates the written file. The commands are + executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='sudo cat {}/test'.format(self.mount_dir), + verify=verify_manila_testing_file) + + def test_manila_share(self): + """Test that a Manila share can be accessed on two instances. + + 1. Create a share + 2. Spawn two servers + 3. Mount it on both + 4. Write a file on one + 5. Read it on the other + 6. Profit + """ + # Create a share + share = self.manila_client.shares.create( + share_type=self.share_type_name, + name=self.share_name, + share_proto=self.share_protocol, + share_network=self.share_network, + size=1) + + # Spawn Servers + instance_1 = self.launch_guest( + guest_name='ins-1', + userdata=self.INSTANCE_USERDATA, + instance_key=self.INSTANCE_KEY) + instance_2 = self.launch_guest( + guest_name='ins-2', + userdata=self.INSTANCE_USERDATA, + instance_key=self.INSTANCE_KEY) + + fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] + fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + + # Wait for the created share to become available before it gets used. + openstack_utils.resource_reaches_status( + self.manila_client.shares, + share.id, + wait_iteration_max_time=120, + stop_after_attempt=2, + expected_status="available", + msg="Waiting for a share to become available") + + # Grant access to the Manila share for both Nova instances. + share.allow(access_type='ip', access=fip_1, access_level='rw') + share.allow(access_type='ip', access=fip_2, access_level='rw') + + ssh_user_name = guest.boot_tests[self.INSTANCE_KEY]['username'] + privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) + share_path = share.export_locations[0] + + # Write a testing file on instance #1 + self._mount_share_on_instance( + fip_1, ssh_user_name, privkey, share_path) + self._write_testing_file_on_instance( + fip_1, ssh_user_name, privkey) + + # Validate the testing file from instance #2 + self._mount_share_on_instance( + fip_2, ssh_user_name, privkey, share_path) + self._validate_testing_file_from_instance( + fip_2, ssh_user_name, privkey) diff --git a/zaza/openstack/charm_tests/manila_ganesha/setup.py b/zaza/openstack/charm_tests/manila_ganesha/setup.py index d2b694e8b..a80495813 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/setup.py +++ b/zaza/openstack/charm_tests/manila_ganesha/setup.py @@ -23,6 +23,9 @@ from manilaclient import client as manilaclient +MANILA_GANESHA_TYPE_NAME = "cephfsnfstype" + + def setup_ganesha_share_type(manila_client=None): """Create a share type for manila with Ganesha. @@ -35,7 +38,7 @@ def setup_ganesha_share_type(manila_client=None): session=keystone_session, client_version='2') manila_client.share_types.create( - name="cephfsnfstype", spec_driver_handles_share_servers=False, + name=MANILA_GANESHA_TYPE_NAME, spec_driver_handles_share_servers=False, extra_specs={ 'vendor_name': 'Ceph', 'storage_protocol': 'NFS', diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 27009bf99..f5d7b6388 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,116 +16,20 @@ """Encapsulate Manila Ganesha testing.""" -from tenacity import Retrying, stop_after_attempt, wait_exponential +from zaza.openstack.charm_tests.manila_ganesha.setup import ( + MANILA_GANESHA_TYPE_NAME, +) -from manilaclient import client as manilaclient +import zaza.openstack.charm_tests.manila.tests as manila_tests -import zaza.openstack.charm_tests.neutron.tests as neutron_tests -import zaza.openstack.charm_tests.nova.utils as nova_utils -import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.configure.guest as guest -import zaza.openstack.utilities.openstack as openstack_utils - -class ManilaGaneshaTests(test_utils.OpenStackBaseTest): +class ManilaGaneshaTests(manila_tests.ManilaBaseTest): """Encapsulate Manila Ganesha tests.""" - RESOURCE_PREFIX = 'zaza-manilatests' - INSTANCE_USERDATA = """#cloud-config -packages: -- nfs-common -""" - @classmethod def setUpClass(cls): """Run class setup for running tests.""" super(ManilaGaneshaTests, cls).setUpClass() - cls.nova_client = ( - openstack_utils.get_nova_session_client(cls.keystone_session)) - cls.manila_client = manilaclient.Client( - session=cls.keystone_session, client_version='2') - - def test_manila_share(self): - """Test that Manila + Ganesha shares can be accessed on two instances. - - 1. create a share - 2. Spawn two servers - 3. mount it on both - 4. write a file on one - 5. read it on the other - 6. profit - """ - # Create a share - share = self.manila_client.shares.create( - share_type='cephfsnfstype', name='cephnfsshare1', - share_proto="nfs", size=1) - - # Spawn Servers - instance_1, instance_2 = self.launch_guests( - userdata=self.INSTANCE_USERDATA) - - fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] - fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] - - # Wait for the created share to become available before it gets used. - openstack_utils.resource_reaches_status( - self.manila_client.shares, - share.id, - wait_iteration_max_time=120, - stop_after_attempt=2, - expected_status="available", - msg="Waiting for a share to become available") - - share.allow(access_type='ip', access=fip_1, access_level='rw') - share.allow(access_type='ip', access=fip_2, access_level='rw') - - # Mount Share - username = guest.boot_tests['bionic']['username'] - password = guest.boot_tests['bionic'].get('password') - privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) - - # Write a file on instance_1 - def verify_setup(stdin, stdout, stderr): - status = stdout.channel.recv_exit_status() - self.assertEqual(status, 0) - - mount_path = share.export_locations[0] - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - openstack_utils.ssh_command( - username, fip_1, 'instance-1', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp ' - '{} /mnt/ceph && ' - 'echo "test" | sudo tee /mnt/ceph/test'.format( - mount_path), - password=password, privkey=privkey, verify=verify_setup) - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - # Setup that file on instance_2 - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp ' - '{} /mnt/ceph' - .format(mount_path), - password=password, privkey=privkey, verify=verify_setup) - - def verify(stdin, stdout, stderr): - status = stdout.channel.recv_exit_status() - self.assertEqual(status, 0) - out = "" - for line in iter(stdout.readline, ""): - out += line - self.assertEqual(out, "test\n") - - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo cat /mnt/ceph/test', - password=password, privkey=privkey, verify=verify) + cls.share_name = 'cephnfsshare1' + cls.share_type_name = MANILA_GANESHA_TYPE_NAME + cls.share_protocol = 'nfs' diff --git a/zaza/openstack/charm_tests/manila_netapp/__init__.py b/zaza/openstack/charm_tests/manila_netapp/__init__.py new file mode 100644 index 000000000..f6e286507 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp setup and testing.""" diff --git a/zaza/openstack/charm_tests/manila_netapp/setup.py b/zaza/openstack/charm_tests/manila_netapp/setup.py new file mode 100644 index 000000000..fa1a671fe --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/setup.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp setup.""" + +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.neutron.setup as neutron_setup + + +MANILA_NETAPP_TYPE_NAME = "netapp-ontap" +MANILA_NETAPP_BACKEND_NAME = "netapp-ontap" + +MANILA_NETAPP_DHSS_TYPE_NAME = "netapp-ontap-dhss" +MANILA_NETAPP_DHSS_BACKEND_NAME = "netapp-ontap-dhss" + +MANILA_NETAPP_SHARE_NET_NAME = "netapp-ontap-share-network" + + +def create_netapp_share_type(manila_client=None): + """Create a share type for Manila with NetApp Data ONTAP driver. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client( + openstack_utils.get_overcloud_keystone_session()) + + manila_client.share_types.create( + name=MANILA_NETAPP_TYPE_NAME, + spec_driver_handles_share_servers=False, + extra_specs={ + 'vendor_name': 'NetApp', + 'share_backend_name': MANILA_NETAPP_BACKEND_NAME, + 'storage_protocol': 'NFS_CIFS', + }) + + +def create_netapp_dhss_share_type(manila_client=None): + """Create a DHSS share type for Manila with NetApp Data ONTAP driver. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client( + openstack_utils.get_overcloud_keystone_session()) + + manila_client.share_types.create( + name=MANILA_NETAPP_DHSS_TYPE_NAME, + spec_driver_handles_share_servers=True, + extra_specs={ + 'vendor_name': 'NetApp', + 'share_backend_name': MANILA_NETAPP_DHSS_BACKEND_NAME, + 'storage_protocol': 'NFS_CIFS', + }) + + +def create_netapp_share_network(manila_client=None): + """Create a Manila share network from the existing provider network. + + This setup function assumes that 'neutron.setup.basic_overcloud_network' + is called to have the proper tenant networks setup. + + The share network will be bound to the provider network configured by + 'neutron.setup.basic_overcloud_network'. + """ + session = openstack_utils.get_overcloud_keystone_session() + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client(session) + + neutron = openstack_utils.get_neutron_session_client(session) + external_net = neutron.find_resource( + 'network', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_net_name']) + external_subnet = neutron.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name']) + + manila_client.share_networks.create( + name=MANILA_NETAPP_SHARE_NET_NAME, + neutron_net_id=external_net['id'], + neutron_subnet_id=external_subnet['id']) diff --git a/zaza/openstack/charm_tests/manila_netapp/tests.py b/zaza/openstack/charm_tests/manila_netapp/tests.py new file mode 100644 index 000000000..f9178ba89 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/tests.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp testing.""" + +from zaza.openstack.charm_tests.manila_netapp.setup import ( + MANILA_NETAPP_TYPE_NAME, + MANILA_NETAPP_DHSS_TYPE_NAME, + MANILA_NETAPP_SHARE_NET_NAME, +) + +import zaza.openstack.charm_tests.manila.tests as manila_tests + + +class ManilaNetAppNFSTest(manila_tests.ManilaBaseTest): + """Encapsulate Manila NetApp NFS test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaNetAppNFSTest, cls).setUpClass() + cls.share_name = 'netapp-ontap-share' + cls.share_type_name = MANILA_NETAPP_TYPE_NAME + cls.share_protocol = 'nfs' + + +class ManilaNetAppDHSSNFSTest(manila_tests.ManilaBaseTest): + """Encapsulate Manila NetApp NFS test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaNetAppDHSSNFSTest, cls).setUpClass() + cls.share_name = 'netapp-ontap-dhss-share' + cls.share_type_name = MANILA_NETAPP_DHSS_TYPE_NAME + cls.share_protocol = 'nfs' + cls.share_network = cls.manila_client.share_networks.find( + name=MANILA_NETAPP_SHARE_NET_NAME) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index d4c89ecbe..4044c5587 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -113,6 +113,56 @@ def get_rw_primary_node(self): if _primary_ip in unit.public_address: return unit + def get_blocked_mysql_routers(self): + """Get blocked mysql routers. + + :returns: List of blocked mysql-router unit names + :rtype: List[str] + """ + # Make sure mysql-router units are up to date + # We cannot assume they are as there is up to a five minute delay + mysql_router_units = [] + for application in self.get_applications_with_substring_in_name( + "mysql-router"): + for unit in zaza.model.get_units(application): + mysql_router_units.append(unit.entity_id) + self.run_update_status_hooks(mysql_router_units) + + # Get up to date status + status = zaza.model.get_status().applications + blocked_mysql_routers = [] + # Check if the units are blocked + for application in self.get_applications_with_substring_in_name( + "mysql-router"): + # Subordinate dance with primary + # There is no satus[applicatoin]["units"] for subordinates + _subordinate_to = status[application].subordinate_to[0] + for appunit in status[_subordinate_to].units: + for subunit in ( + status[_subordinate_to]. + units[appunit].subordinates.keys()): + if "blocked" in ( + status[_subordinate_to].units[appunit]. + subordinates[subunit].workload_status.status): + blocked_mysql_routers.append(subunit) + return blocked_mysql_routers + + def restart_blocked_mysql_routers(self): + """Restart blocked mysql routers. + + :returns: None + :rtype: None + """ + # Check for blocked mysql-router units + blocked_mysql_routers = self.get_blocked_mysql_routers() + for unit in blocked_mysql_routers: + logging.warning( + "Restarting blocked mysql-router unit {}" + .format(unit)) + zaza.model.run_on_unit( + unit, + "systemctl restart {}".format(unit.rpartition("/")[0])) + class MySQLCommonTests(MySQLBaseTest): """Common mysql charm tests.""" @@ -169,6 +219,15 @@ def test_920_pause_resume(self): """ with self.pause_resume(self.services): logging.info("Testing pause resume") + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # If there are any blocekd mysql routers restart them. + self.restart_blocked_mysql_routers() + assert not self.get_blocked_mysql_routers(), ( + "Should no longer be blocked mysql-router units") + logging.info("Passed pause and resume test.") @@ -589,6 +648,10 @@ def test_100_reboot_cluster_from_complete_outage(self): unit.entity_id, 'blocked') + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") # We do not know which unit has the most up to date data @@ -753,13 +816,22 @@ def test_800_remove_leader(self): logging.info("Scale in test: remove leader") leader, nons = self.get_leaders_and_non_leaders() leader_unit = zaza.model.get_unit_from_name(leader) - zaza.model.destroy_unit(self.application_name, leader) - - logging.info("Wait until unit is in waiting state ...") - zaza.model.block_until_unit_wl_status(nons[0], "waiting") + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, leader) + + logging.info("Wait until all only 2 units ...") + zaza.model.block_until_unit_count(self.application, 2) + + logging.info("Wait until all units are cluster incomplete ...") + zaza.model.block_until_wl_status_info_starts_with( + self.application, "'cluster' incomplete") + + # Show status + logging.info(self.get_cluster_status()) logging.info( "Removing old unit from cluster: {} " @@ -786,6 +858,9 @@ def test_801_add_unit(self): logging.info("Adding unit after removed unit ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -801,6 +876,9 @@ def test_802_add_unit(self): logging.info("Adding unit after full cluster ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 4 units ...") + zaza.model.block_until_unit_count(self.application, 4) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -810,19 +888,26 @@ def test_803_remove_fourth(self): We start with a four node full cluster, remove one, down to a three node full cluster. """ - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - leader, nons = self.get_leaders_and_non_leaders() non_leader_unit = zaza.model.get_unit_from_name(nons[0]) - zaza.model.destroy_unit(self.application_name, nons[0]) + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, nons[0]) + logging.info("Scale in test: back down to three") + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + + logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(non_leader_unit.public_address)) @@ -835,3 +920,109 @@ def test_803_remove_fourth(self): assert action.data.get("results") is not None, ( "Remove instance action failed: No results: {}" .format(action.data)) + + +class MySQLInnoDBClusterPartitionTest(MySQLBaseTest): + """MySQL partition handling.""" + + def test_850_force_quorum_using_partition_of(self): + """Force quorum using partition of instance with given address. + + After outage, cluster can end up without quorum. Force it. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 + mysql_units = [unit for unit in zaza.model.get_units(self.application)] + no_of_units = len(mysql_units) + for index, unit in enumerate(mysql_units): + next_unit = mysql_units[(index+1) % no_of_units] + ip_address = next_unit.public_address + cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) + zaza.model.async_run_on_unit(unit, cmd) + + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked', + negate_match=True) + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Execute force-quorum-using-partition-of action ...") + + # Select "quorum leader" unit + leader_unit = mysql_units[0] + action = zaza.model.run_action( + leader_unit.entity_id, + "force-quorum-using-partition-of", + action_params={ + "address": leader_unit.public_address, + 'i-really-mean-it': True + }) + + assert action.data.get("results") is not None, ( + "Force quorum using partition of action failed: {}" + .format(action.data)) + logging.debug( + "Results from running 'force-quorum' command ...\n{}".format( + action.data)) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) + + logging.info("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) + + +class MySQLRouterTests(test_utils.OpenStackBaseTest): + """MySQL Router Tests.""" + + @classmethod + def setUpClass(cls, application_name="keystone-mysql-router"): + """Run class setup for running mysql-router tests.""" + super().setUpClass(application_name=application_name) + cls.application = application_name + cls.services = ["mysqlrouter"] + # Config file affected by juju set config change + cls.conf_file = ( + "/var/lib/mysql/{}-mysql-router/mysqlrouter.conf" + .format(application_name)) + + def test_910_restart_on_config_change(self): + """Checking restart happens on config change. + + Change max connections and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"ttl": ".5"} + set_alternate = {"ttl": "7"} + + # Make config change, check for service restarts + logging.info("Setting TTL ...") + self.restart_on_changed( + self.conf_file, + set_default, + set_alternate, + {}, {}, + self.services) + logging.info("Passed restart on changed test.") diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py index 1fe5114fc..75da5d97e 100644 --- a/zaza/openstack/charm_tests/mysql/utils.py +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -19,8 +19,16 @@ async def complete_cluster_series_upgrade(): """Run the complete-cluster-series-upgrade action on the lead unit.""" - # TODO: Make this work across either mysql or percona-cluster names - await model.async_run_action_on_leader( - 'mysql', - 'complete-cluster-series-upgrade', - action_params={}) + # Note that some models use mysql as the application name, and other's use + # percona-cluster. Try mysql first, and if it doesn't exist, then try + # percona-cluster instead. + try: + await model.async_run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + except KeyError: + await model.async_run_action_on_leader( + 'percona-cluster', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index f65b87adb..a1d1dd402 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -15,6 +15,7 @@ """Setup for Neutron deployments.""" import functools +import logging from zaza.openstack.configure import ( network, @@ -89,12 +90,25 @@ def basic_overcloud_network(limit_gws=None): 'configure_gateway_ext_port_use_juju_wait', True) # Handle network for OpenStack-on-OpenStack scenarios - if juju_utils.get_provider_type() == "openstack": + provider_type = juju_utils.get_provider_type() + if provider_type == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session() network.setup_gateway_ext_port(network_config, keystone_session=undercloud_ks_sess, - limit_gws=None, + limit_gws=limit_gws, use_juju_wait=use_juju_wait) + elif provider_type == "maas": + # NOTE(fnordahl): After validation of the MAAS+Netplan Open vSwitch + # integration support, we would most likely want to add multiple modes + # of operation with MAAS. + # + # Perform charm based OVS configuration + openstack_utils.configure_charmed_openstack_on_maas( + network_config, limit_gws=limit_gws) + else: + logging.warning('Unknown Juju provider type, "{}", will not perform' + ' charm network configuration.' + .format(provider_type)) # Confugre the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index bebe3b609..54420e8b2 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -592,6 +592,58 @@ def test_901_pause_and_resume(self): logging.info('Testing pause resume') +class NeutronBridgePortMappingTest(NeutronPluginApiSharedTests): + """Test correct handling of network-bridge-port mapping functionality.""" + + def test_600_conflict_data_ext_ports(self): + """Verify proper handling of conflict between data-port and ext-port. + + Configuring ext-port and data-port at the same time should make the + charm to enter "blocked" state. After unsetting ext-port charm should + be active again. + """ + if self.application_name not in ["neutron-gateway", + "neutron-openvswitch"]: + logging.debug("Skipping test, charm under test is not " + "neutron-gateway or neutron-openvswitch") + return + + current_data_port = zaza.model.get_application_config( + self.application_name).get("data-port").get("value", "") + current_ext_port = zaza.model.get_application_config( + self.application_name).get("ext-port").get("value", "") + logging.debug("Current data-port: '{}'".format(current_data_port)) + logging.debug("Current data-port: '{}'".format(current_ext_port)) + + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + current_state = test_config.get("target_deploy_status", {}) + blocked_state = copy.deepcopy(current_state) + blocked_state[self.application_name] = { + "workload-status": "blocked", + "workload-status-message": + "ext-port set when data-port set: see config.yaml"} + + logging.info("Setting conflicting ext-port and data-port options") + zaza.model.set_application_config( + self.application_name, {"data-port": "br-phynet43:eth43", + "ext-port": "br-phynet43:eth43"}) + zaza.model.wait_for_application_states(states=blocked_state) + + # unset ext-port and wait for app state to return to active + logging.info("Unsetting conflicting ext-port option") + zaza.model.set_application_config( + self.application_name, {"ext-port": ""}) + zaza.model.wait_for_application_states(states=current_state) + + # restore original config + zaza.model.set_application_config( + self.application_name, {'data-port': current_data_port, + 'ext-port': current_ext_port}) + zaza.model.wait_for_application_states(states=current_state) + logging.info('OK') + + class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): """Test 'ovs-vsctl'-related functionality on Neutron charms.""" @@ -610,7 +662,8 @@ def test_800_ovs_bridges_are_managed_by_us(self): unit.name, bridge_name ) + ' is marked as managed by us' ) - expected_external_id = 'charm-neutron-gateway=managed' + expected_external_id = 'charm-{}=managed'.format( + self.application_name) actual_external_id = zaza.model.run_on_unit( unit.entity_id, 'ovs-vsctl br-get-external-id {}'.format(bridge_name), @@ -952,3 +1005,47 @@ def test_gateway_failure(self): uc_neutron_client, gateway_hostname) self.check_connectivity(instance_1, instance_2) + + +class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='neutron-openvswitch') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + +class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='neutron-gateway') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which requires a restart + self.run_charm_change_restart_test( + 'neutron-l3-agent', + '/etc/neutron/neutron.conf') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def check_clear_hooks(self): + """Gateway does not defer hooks so noop.""" + return diff --git a/zaza/openstack/charm_tests/nova/setup.py b/zaza/openstack/charm_tests/nova/setup.py index c556c9762..7f8ee5372 100644 --- a/zaza/openstack/charm_tests/nova/setup.py +++ b/zaza/openstack/charm_tests/nova/setup.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for configureing nova.""" +"""Code for configuring nova.""" + +import tenacity import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.utilities import ( @@ -21,6 +23,9 @@ import zaza.openstack.charm_tests.nova.utils as nova_utils +@tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) def create_flavors(nova_client=None): """Create basic flavors. @@ -43,6 +48,9 @@ def create_flavors(nova_client=None): flavorid=nova_utils.FLAVORS[flavor]['flavorid']) +@tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) def manage_ssh_key(nova_client=None): """Create basic flavors. diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index c4dd5b480..66ca957b6 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -19,6 +19,8 @@ import json import logging import unittest +from configparser import ConfigParser +from time import sleep import zaza.model import zaza.openstack.charm_tests.glance.setup as glance_setup @@ -38,37 +40,294 @@ def launch_instance(self, instance_key): zaza.openstack.configure.guest.launch_instance(instance_key) -class CirrosGuestCreateTest(BaseGuestCreateTest): +class CirrosGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a cirros image.""" def test_launch_small_instance(self): """Launch a cirros instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.CIRROS_IMAGE_NAME) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() -class LTSGuestCreateTest(BaseGuestCreateTest): + +class LTSGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" def test_launch_small_instance(self): """Launch a Bionic instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) + + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() -class LTSGuestCreateVolumeBackedTest(BaseGuestCreateTest): +class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" def test_launch_small_instance(self): """Launch a Bionic instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, use_boot_volume=True) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + + +class NovaCommonTests(test_utils.OpenStackBaseTest): + """nova-compute and nova-cloud-controller common tests.""" + + XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') + XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') + XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') + BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') + BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') + BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') + + @classmethod + def setUpClass(cls): + """Run class setup for running nova-cloud-controller tests.""" + super(NovaCommonTests, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release() + + def _test_pci_alias_config(self, app_name, service_list): + logging.info('Checking pci aliases in nova config...') + + # Expected default and alternate values + current_value = zaza.model.get_application_config( + app_name)['pci-alias'] + try: + current_value = current_value['value'] + except KeyError: + current_value = None + new_value = '[{}, {}]'.format( + json.dumps({ + 'name': 'IntelNIC', + 'capability_type': 'pci', + 'product_id': '1111', + 'vendor_id': '8086', + 'device_type': 'type-PF' + }, sort_keys=True), + json.dumps({ + 'name': ' Cirrus Logic ', + 'capability_type': 'pci', + 'product_id': '0ff2', + 'vendor_id': '10de', + 'device_type': 'type-PCI' + }, sort_keys=True)) + + set_default = {'pci-alias': current_value} + set_alternate = {'pci-alias': new_value} + + expected_conf_section = 'pci' + expected_conf_key = 'alias' -class NovaCompute(test_utils.OpenStackBaseTest): + default_entry = {expected_conf_section: {}} + alternate_entry = {expected_conf_section: { + expected_conf_key: [ + ('{"capability_type": "pci", "device_type": "type-PF", ' + '"name": "IntelNIC", "product_id": "1111", ' + '"vendor_id": "8086"}'), + ('{"capability_type": "pci", "device_type": "type-PCI", ' + '"name": " Cirrus Logic ", "product_id": "0ff2", ' + '"vendor_id": "10de"}')]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting config on {} to {}'.format(app_name, set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + service_list) + + +class CloudActions(test_utils.OpenStackBaseTest): + """Test actions from actions/cloud.py.""" + + def fetch_nova_service_hostname(self, unit_name): + """ + Fetch hostname used to register with nova-cloud-controller. + + When nova-compute registers with nova-cloud-controller it uses either + config variable from '/etc/nova/nova.conf` or host's hostname to + identify itself. We need to fetch this value directly from the unit, + otherwise it's not possible to correlate entries from + `nova service-list` with nova-compute units. + + :param unit_name: nova-compute unit name. + :return: hostname used when registering to cloud-controller + """ + nova_cfg = ConfigParser() + + result = zaza.model.run_on_unit(unit_name, + 'cat /etc/nova/nova.conf') + nova_cfg.read_string(result['Stdout']) + + try: + nova_service_name = nova_cfg['DEFAULT']['host'] + except KeyError: + # Fallback to hostname if 'host' variable is not present in the + # config + result = zaza.model.run_on_unit(unit_name, 'hostname') + nova_service_name = result['Stdout'].rstrip('\n') + + if not nova_service_name: + self.fail("Failed to fetch nova service name from" + " nova-compute unit.") + return nova_service_name + + def test_940_enable_disable_actions(self): + """Test disable/enable actions on nova-compute units.""" + nova_units = zaza.model.get_units('nova-compute', + model_name=self.model_name) + + # Check that nova-compute services are enabled before testing + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'enabled') + + # Run 'disable' action on units + zaza.model.run_action_on_units([unit.name for unit in nova_units], + 'disable') + + # Check action results via nova API + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'disabled') + + # Run 'enable' action on units + zaza.model.run_action_on_units([unit.name for unit in nova_units], + 'enable') + + # Check action results via nova API + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'enabled') + + def test_950_instance_count_action(self): + """Test that action 'instance-count' returns expected values.""" + def check_instance_count(expect_count, unit_name): + """Assert that unit with 'unit_name' has 'expect_count' of VMs. + + :param expect_count: How many VMs are expected to be running + :param unit_name: Name of the target nova-compute unit + :return: None + :raises AssertionError: If result of the 'instance-count' action + does not match 'expect_count'. + """ + logging.debug('Running "instance-count" action on unit "{}".' + 'Expecting result: {}'.format(unit_name, + expect_count)) + result = zaza.model.run_action(unit_name, 'instance-count') + self.assertEqual(result.status, 'completed') + instances = result.data.get('results', {}).get('instance-count') + self.assertEqual(instances, str(expect_count)) + + nova_unit = zaza.model.get_units('nova-compute', + model_name=self.model_name)[0] + + check_instance_count(0, nova_unit.entity_id) + + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) + + check_instance_count(1, nova_unit.entity_id) + + self.resource_cleanup() + + def test_960_remove_from_cloud_actions(self): + """Test actions remove-from-cloud and register-to-cloud. + + Note (martin-kalcok): This test requires that nova-compute unit is not + running any VMs. If there are any leftover VMs from previous tests, + action `remove-from-cloud` will fail. + """ + def wait_for_nova_compute_count(expected_count): + """Wait for expected number of nova compute services to be present. + + Returns True or False based on whether the expected number of nova + compute services was reached within the timeout. Checks are + performed every 10 second in the span of maximum 5 minutes. + """ + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == expected_count: + return True + sleep_timeout = 10 + return False + + all_units = zaza.model.get_units('nova-compute', + model_name=self.model_name) + + unit_to_remove = all_units[0] + + service_name = self.fetch_nova_service_hostname(unit_to_remove.name) + + registered_nova_services = self.nova_client.services.list( + host=service_name, binary='nova-compute') + + service_count = len(registered_nova_services) + if service_count < 1: + self.fail("Unit '{}' has no nova-compute services registered in" + " nova-cloud-controller".format(unit_to_remove.name)) + elif service_count > 1: + self.fail("Unexpected number of nova-compute services registered" + " in nova-cloud controller. Expecting: 1, found: " + "{}".format(service_count)) + + # run action remove-from-cloud and wait for the results in + # nova-cloud-controller + zaza.model.run_action_on_units([unit_to_remove.name], + 'remove-from-cloud', + raise_on_failure=True) + + # Wait for nova-compute service to be removed from the + # nova-cloud-controller + if not wait_for_nova_compute_count(0): + self.fail("nova-compute service was not unregistered from the " + "nova-cloud-controller as expected.") + + # run action register-to-cloud to revert previous action + # and wait for the results in nova-cloud-controller + zaza.model.run_action_on_units([unit_to_remove.name], + 'register-to-cloud', + raise_on_failure=True) + + if not wait_for_nova_compute_count(1): + self.fail("nova-compute service was not re-registered to the " + "nova-cloud-controller as expected.") + + +class NovaCompute(NovaCommonTests): """Run nova-compute specific tests.""" + def test_311_pci_alias_config_compute(self): + """Verify that the pci alias data is rendered properly. + + Change pci-alias and assert that change propagates to the correct + file and that services are restarted as a result + """ + # We are not touching the behavior of anything older than QUEENS + if self.current_release >= self.XENIAL_QUEENS: + self._test_pci_alias_config("nova-compute", ['nova-compute']) + def test_500_hugepagereport_action(self): """Test hugepagereport action.""" for unit in zaza.model.get_units('nova-compute', @@ -152,21 +411,87 @@ def test_930_check_virsh_default_network(self): self.assertFalse(int(run['Code']) == 0) -class NovaCloudController(test_utils.OpenStackBaseTest): - """Run nova-cloud-controller specific tests.""" +class NovaComputeActionTest(test_utils.OpenStackBaseTest): + """Run nova-compute specific tests. - XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') - XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') - XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') - BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') - BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') - BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') + Add this test class for new nova-compute action + to avoid breaking older version + """ - @classmethod - def setUpClass(cls): - """Run class setup for running nova-cloud-controller tests.""" - super(NovaCloudController, cls).setUpClass() - cls.current_release = openstack_utils.get_os_release() + def test_virsh_audit_action(self): + """Test virsh-audit action.""" + for unit in zaza.model.get_units('nova-compute', + model_name=self.model_name): + logging.info('Running `virsh-audit` action' + ' on unit {}'.format(unit.entity_id)) + action = zaza.model.run_action( + unit.entity_id, + 'virsh-audit', + model_name=self.model_name, + action_params={}) + if "failed" in action.data["status"]: + raise Exception( + "The action failed: {}".format(action.data["message"])) + + +class NovaCloudControllerActionTest(test_utils.OpenStackBaseTest): + """Run nova-cloud-controller specific tests. + + Add this test class for new nova-cloud-controller action + to avoid breaking older version. + """ + + def test_sync_compute_az_action(self): + """Test sync-compute-availability-zones action.""" + juju_units_az_map = {} + compute_config = zaza.model.get_application_config('nova-compute') + default_az = compute_config['default-availability-zone']['value'] + use_juju_az = compute_config['customize-failure-domain']['value'] + + for unit in zaza.model.get_units('nova-compute', + model_name=self.model_name): + zone = default_az + if use_juju_az: + result = zaza.model.run_on_unit(unit.name, + 'echo $JUJU_AVAILABILITY_ZONE', + model_name=self.model_name, + timeout=60) + self.assertEqual(int(result['Code']), 0) + juju_az = result['Stdout'].strip() + if juju_az: + zone = juju_az + + juju_units_az_map[unit.public_address] = zone + continue + + session = openstack_utils.get_overcloud_keystone_session() + nova = openstack_utils.get_nova_session_client(session) + + result = zaza.model.run_action_on_leader( + 'nova-cloud-controller', + 'sync-compute-availability-zones', + model_name=self.model_name) + + # For validating the action results, we simply want to validate that + # the action was completed and we have something in the output. The + # functional validation really occurs below, in that the hosts are + # checked to be in the appropriate host aggregates. + self.assertEqual(result.status, 'completed') + self.assertNotEqual('', result.results['output']) + + unique_az_list = list(set(juju_units_az_map.values())) + aggregates = nova.aggregates.list() + self.assertEqual(len(aggregates), len(unique_az_list)) + for unit_address in juju_units_az_map: + az = juju_units_az_map[unit_address] + aggregate = nova.aggregates.find( + name='{}_az'.format(az), availability_zone=az) + hypervisor = nova.hypervisors.find(host_ip=unit_address) + self.assertIn(hypervisor.hypervisor_hostname, aggregate.hosts) + + +class NovaCloudController(NovaCommonTests): + """Run nova-cloud-controller specific tests.""" @property def services(self): @@ -263,70 +588,13 @@ def test_302_api_rate_limiting_is_enabled(self): 'filter:legacy_ratelimit': { 'limits': ["( POST, '*', .*, 9999, MINUTE );"]}}) - def test_310_pci_alias_config(self): + def test_310_pci_alias_config_ncc(self): """Verify that the pci alias data is rendered properly. Change pci-alias and assert that change propagates to the correct file and that services are restarted as a result """ - logging.info('Checking pci aliases in nova config...') - - # Expected default and alternate values - current_value = zaza.model.get_application_config( - 'nova-cloud-controller')['pci-alias'] - try: - current_value = current_value['value'] - except KeyError: - current_value = None - new_value = '[{}, {}]'.format( - json.dumps({ - 'name': 'IntelNIC', - 'capability_type': 'pci', - 'product_id': '1111', - 'vendor_id': '8086', - 'device_type': 'type-PF' - }, sort_keys=True), - json.dumps({ - 'name': ' Cirrus Logic ', - 'capability_type': 'pci', - 'product_id': '0ff2', - 'vendor_id': '10de', - 'device_type': 'type-PCI' - }, sort_keys=True)) - - set_default = {'pci-alias': current_value} - set_alternate = {'pci-alias': new_value} - - expected_conf_section = 'DEFAULT' - expected_conf_key = 'pci_alias' - if self.current_release >= self.XENIAL_OCATA: - expected_conf_section = 'pci' - expected_conf_key = 'alias' - - default_entry = {expected_conf_section: {}} - alternate_entry = {expected_conf_section: { - expected_conf_key: [ - ('{"capability_type": "pci", "device_type": "type-PF", ' - '"name": "IntelNIC", "product_id": "1111", ' - '"vendor_id": "8086"}'), - ('{"capability_type": "pci", "device_type": "type-PCI", ' - '"name": " Cirrus Logic ", "product_id": "0ff2", ' - '"vendor_id": "10de"}')]}} - - # Config file affected by juju set config change - conf_file = '/etc/nova/nova.conf' - - # Make config change, check for service restarts - logging.info( - 'Setting config on nova-cloud-controller to {}'.format( - set_alternate)) - self.restart_on_changed( - conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, - self.services) + self._test_pci_alias_config("nova-cloud-controller", self.services) def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -469,17 +737,24 @@ def test_security_checklist(self): # Changes fixing the below expected failures will be made following # this initial work to get validation in. There will be bugs targeted # to each one and resolved independently where possible. - expected_failures = [ - 'is-volume-encryption-enabled', - 'validate-uses-tls-for-glance', - 'validate-uses-tls-for-keystone', ] expected_passes = [ 'validate-file-ownership', 'validate-file-permissions', 'validate-uses-keystone', ] + tls_checks = [ + 'validate-uses-tls-for-glance', + 'validate-uses-tls-for-keystone', + ] + if zaza.model.get_relation_id( + 'nova-cloud-controller', + 'vault', + remote_interface_name='certificates'): + expected_passes.extend(tls_checks) + else: + expected_failures.extend(tls_checks) for unit in zaza.model.get_units(self.application_name, model_name=self.model_name): @@ -493,4 +768,4 @@ def test_security_checklist(self): action_params={}), expected_passes, expected_failures, - expected_to_pass=False) + expected_to_pass=not len(expected_failures)) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 677c36863..b04b69660 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -25,6 +25,9 @@ import zaza.openstack.utilities.openstack as openstack import zaza.openstack.configure.guest +import zaza.openstack.charm_tests.nova.setup as nova_setup +import zaza.openstack.charm_tests.nova.utils as nova_utils + def ensure_lts_images(): """Ensure that bionic and focal images are available for the tests.""" @@ -51,13 +54,32 @@ def add_amphora_image(image_url=None): def configure_octavia(): - """Do mandatory post deployment configuration of Octavia.""" - # Tell Octavia charm it is safe to create cloud resources - logging.info('Running `configure-resources` action on Octavia leader unit') - zaza.model.run_action_on_leader( - 'octavia', - 'configure-resources', - action_params={}) + """Do post deployment configuration and initialization of Octavia. + + Certificates for the private Octavia worker <-> Amphorae communication must + be generated and set trough charm configuration. + + The optional SSH configuration options are set to enable debug and log + collection from Amphorae, we will use the same keypair as Zaza uses for + instance creation. + + The `configure-resources` action must be run to have the charm create + in-cloud resources such as management network and associated ports and + security groups. + """ + # Set up Nova client to create/retrieve keypair for Amphora debug purposes. + # + # We reuse the Nova setup code for this and in most cases the test + # declaration will already defined that the Nova manage_ssh_key setup + # helper to run before we get here. Re-run here to make sure this setup + # function can be used separately, manage_ssh_key is idempotent. + keystone_session = openstack.get_overcloud_keystone_session() + nova_client = openstack.get_nova_session_client( + keystone_session) + nova_setup.manage_ssh_key(nova_client) + ssh_public_key = openstack.get_public_key( + nova_client, nova_utils.KEYPAIR_NAME) + # Generate certificates for controller/load balancer instance communication (issuing_cakey, issuing_cacert) = cert.generate_cert( 'OSCI Zaza Issuer', @@ -71,7 +93,7 @@ def configure_octavia(): issuer_name='OSCI Zaza Octavia Controller', signing_key=controller_cakey) controller_bundle = controller_cert + controller_key - cert_config = { + charm_config = { 'lb-mgmt-issuing-cacert': base64.b64encode( issuing_cacert).decode('utf-8'), 'lb-mgmt-issuing-ca-private-key': base64.b64encode( @@ -81,22 +103,45 @@ def configure_octavia(): controller_cacert).decode('utf-8'), 'lb-mgmt-controller-cert': base64.b64encode( controller_bundle).decode('utf-8'), + 'amp-ssh-key-name': 'octavia', + 'amp-ssh-pub-key': base64.b64encode( + bytes(ssh_public_key, 'utf-8')).decode('utf-8'), } - logging.info('Configuring certificates for mandatory Octavia ' - 'client/server authentication ' - '(client being the ``Amphorae`` load balancer instances)') + + # Tell Octavia charm it is safe to create cloud resources, we do this now + # because the workload status will be checked on config-change and it gets + # a bit complicated to augment test config to accept 'blocked' vs. 'active' + # in the various stages. + logging.info('Running `configure-resources` action on Octavia leader unit') + zaza.model.run_action_on_leader( + 'octavia', + 'configure-resources', + action_params={}) # Our expected workload status will change after we have configured the # certificates test_config = zaza.charm_lifecycle.utils.get_charm_config() del test_config['target_deploy_status']['octavia'] + logging.info('Configuring certificates for mandatory Octavia ' + 'client/server authentication ' + '(client being the ``Amphorae`` load balancer instances)') + _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() _singleton.setUpClass(application_name='octavia') - with _singleton.config_change(cert_config, cert_config): + with _singleton.config_change(charm_config, charm_config): # wait for configuration to be applied then return pass + # Should we consider making the charm attempt to create this key on + # config-change? + logging.info('Running `configure-resources` action again to ensure ' + 'Octavia Nova SSH key pair is created after config change.') + zaza.model.run_action_on_leader( + 'octavia', + 'configure-resources', + action_params={}) + def centralized_fip_network(): """Create network with centralized router for connecting lb and fips. diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index eb0b21776..52083ab39 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,6 +25,8 @@ import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack.utilities import ObjectRetrierWraps + class CharmOperationTest(test_utils.OpenStackBaseTest): """Charm operation tests.""" @@ -63,12 +65,12 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() - cls.keystone_client = openstack_utils.get_keystone_session_client( - cls.keystone_session) - cls.neutron_client = openstack_utils.get_neutron_session_client( - cls.keystone_session) - cls.octavia_client = openstack_utils.get_octavia_session_client( - cls.keystone_session) + cls.keystone_client = ObjectRetrierWraps( + openstack_utils.get_keystone_session_client(cls.keystone_session)) + cls.neutron_client = ObjectRetrierWraps( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + cls.octavia_client = ObjectRetrierWraps( + openstack_utils.get_octavia_session_client(cls.keystone_session)) cls.RESOURCE_PREFIX = 'zaza-octavia' # NOTE(fnordahl): in the event of a test failure we do not want to run @@ -280,7 +282,13 @@ def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, lambda x: octavia_client.member_show( pool_id=pool_id, member_id=x), member_id, - operating_status='ONLINE' if monitor else '') + operating_status='') + # Temporarily disable this check until we figure out why + # operational_status sometimes does not become 'ONLINE' + # while the member does indeed work and the subsequent + # retrieval of payload through loadbalancer is successful + # ref LP: #1896729. + # operating_status='ONLINE' if monitor else '') logging.info(resp) return lb diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index d3fc625bd..5cd341c8c 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -67,7 +67,6 @@ def _login(dashboard_url, domain, username, password, cafile=None): # start session, get csrftoken client = requests.session() client.get(auth_url, verify=cafile) - if 'csrftoken' in client.cookies: csrftoken = client.cookies['csrftoken'] else: @@ -163,7 +162,60 @@ def _do_request(request, cafile=None): return urllib.request.urlopen(request, cafile=cafile) -class OpenStackDashboardTests(test_utils.OpenStackBaseTest): +class OpenStackDashboardBase(): + """Mixin for interacting with Horizon.""" + + def get_base_url(self): + """Return the base url for http(s) requests. + + :returns: URL + :rtype: str + """ + vip = (zaza_model.get_application_config(self.application_name) + .get("vip").get("value")) + if vip: + ip = vip + else: + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + ip = unit.public_address + + logging.debug("Dashboard ip is:{}".format(ip)) + scheme = 'http' + if self.use_https: + scheme = 'https' + url = '{}://{}'.format(scheme, ip) + return url + + def get_horizon_url(self): + """Return the url for acccessing horizon. + + :returns: Horizon URL + :rtype: str + """ + url = '{}/horizon'.format(self.get_base_url()) + logging.info("Horizon URL is: {}".format(url)) + return url + + @property + def use_https(self): + """Whether dashboard is using https. + + :returns: Whether dashboard is using https + :rtype: boolean + """ + use_https = False + vault_relation = zaza_model.get_relation_id( + self.application, + 'vault', + remote_interface_name='certificates') + if vault_relation: + use_https = True + return use_https + + +class OpenStackDashboardTests(test_utils.OpenStackBaseTest, + OpenStackDashboardBase): """Encapsulate openstack dashboard charm tests.""" @classmethod @@ -171,13 +223,6 @@ def setUpClass(cls): """Run class setup for running openstack dashboard charm tests.""" super(OpenStackDashboardTests, cls).setUpClass() cls.application = 'openstack-dashboard' - cls.use_https = False - vault_relation = zaza_model.get_relation_id( - cls.application, - 'vault', - remote_interface_name='certificates') - if vault_relation: - cls.use_https = True def test_050_local_settings_permissions_regression_check_lp1755027(self): """Assert regression check lp1755027. @@ -302,39 +347,6 @@ def crude_py_parse(self, file_contents, expected): mismatches.append(msg) return mismatches - def get_base_url(self): - """Return the base url for http(s) requests. - - :returns: URL - :rtype: str - """ - vip = (zaza_model.get_application_config(self.application_name) - .get("vip").get("value")) - if vip: - ip = vip - else: - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) - ip = unit.public_address - - logging.debug("Dashboard ip is:{}".format(ip)) - scheme = 'http' - if self.use_https: - scheme = 'https' - url = '{}://{}'.format(scheme, ip) - logging.debug("Base URL is: {}".format(url)) - return url - - def get_horizon_url(self): - """Return the url for acccessing horizon. - - :returns: Horizon URL - :rtype: str - """ - url = '{}/horizon'.format(self.get_base_url()) - logging.info("Horizon URL is: {}".format(url)) - return url - def test_400_connection(self): """Test that dashboard responds to http request. @@ -395,17 +407,6 @@ def test_404_connection(self): self.assertEqual(e.code, 404, msg) logging.info('OK') - def test_501_security_checklist_action(self): - """Verify expected result on a default install. - - Ported from amulet tests. - """ - logging.info("Testing security-checklist") - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - action = zaza_model.run_action(unit_name, 'security-checklist') - assert action.data.get(u"status") == "failed", \ - "Security check is expected to not pass by default" - def test_900_restart_on_config_change(self): """Verify that the specified services are restarted on config changed. @@ -450,7 +451,8 @@ def test_910_pause_and_resume(self): logging.info("Testing pause resume") -class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): +class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, + OpenStackDashboardBase): """Test the policyd override using the dashboard.""" good = { @@ -476,6 +478,7 @@ def setUpClass(cls, application_name=None): super(OpenStackDashboardPolicydTests, cls).setUpClass( application_name="openstack-dashboard") cls.application_name = "openstack-dashboard" + cls.application = cls.application_name def get_client_and_attempt_operation(self, ip): """Attempt to list users on the openstack-dashboard service. @@ -500,9 +503,51 @@ def get_client_and_attempt_operation(self, ip): username = 'admin', password = overcloud_auth['OS_PASSWORD'], client, response = _login( - unit.public_address, domain, username, password) + self.get_horizon_url(), domain, username, password) # now attempt to get the domains page _url = self.url.format(unit.public_address) result = client.get(_url) if result.status_code == 403: raise policyd.PolicydOperationFailedException("Not authenticated") + + +class SecurityTests(test_utils.OpenStackBaseTest, + OpenStackDashboardBase): + """Openstack-dashboard security tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running openstack-dashboard SecurityTests.""" + super(SecurityTests, cls).setUpClass() + + def test_security_checklist(self): + """Verify expected state with security checklist.""" + logging.info("Testing security checklist.") + + expected_failures = [ + 'csrf_cookie_set', + 'disable_password_reveal', + 'disallow-iframe-embed', + 'password-validator-is-not-default', + 'securie_proxy_ssl_header_is_set', + 'session_cookie-httponly', + 'session-cookie-store', + ] + expected_passes = [ + 'disable_password_autocomplete', + 'enforce-password-check', + 'validate-file-ownership', + 'validate-file-permissions' + ] + + logging.info('Running `security-checklist` action' + ' on {} leader'.format(self.application_name)) + test_utils.audit_assertions( + zaza_model.run_action_on_leader( + self.application_name, + 'security-checklist', + model_name=self.model_name, + action_params={}), + expected_passes, + expected_failures, + expected_to_pass=False) diff --git a/zaza/openstack/charm_tests/security/__init__.py b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py similarity index 87% rename from zaza/openstack/charm_tests/security/__init__.py rename to zaza/openstack/charm_tests/openstack_upgrade/__init__.py index ec47696c4..9a1ca538a 100644 --- a/zaza/openstack/charm_tests/security/__init__.py +++ b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Canonical Ltd. +# Copyright 2020 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test security checklist.""" +"""Code for testing openstack upgrades.""" diff --git a/zaza/openstack/charm_tests/openstack_upgrade/tests.py b/zaza/openstack/charm_tests/openstack_upgrade/tests.py new file mode 100644 index 000000000..82c7fbb99 --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/tests.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for OpenStack Upgrade.""" + +import logging +import unittest + +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, + openstack as openstack_utils, + openstack_upgrade as openstack_upgrade, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest + + +class OpenStackUpgradeVMLaunchBase(object): + """A base class to peform a simple validation on the cloud. + + This wraps an OpenStack upgrade with a VM launch before and after the + upgrade. + + This test requires a full OpenStack including at least: keystone, glance, + nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and + neutron-openvswitch. + + This class should be used as a base class to the upgrade 'test'. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenStackUpgradeMixin setUpClass") + super().setUpClass() + cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() + + def test_100_validate_pre_openstack_upgrade_cloud(self): + """Validate pre openstack upgrade.""" + logging.info("Validate pre-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_500_validate_openstack_upgraded_cloud(self): + """Validate post openstack upgrade.""" + logging.info("Validate post-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +class WaitForMySQL(unittest.TestCase): + """Helper test to wait on mysql-innodb-cluster to be fully ready. + + In practice this means that there is at least on R/W unit available. + Sometimes, after restarting units in the mysql-innodb-cluster, all the + units are R/O until the cluster picks the R/W unit. + """ + + @classmethod + def setUpClass(cls): + """Set up class.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_100_wait_for_happy_mysql_innodb_cluster(self): + """Wait for mysql cluster to have at least one R/W node.""" + logging.info("Starting wait for an R/W unit.") + openstack_upgrade.block_until_mysql_innodb_cluster_has_rw() + logging.info("Done .. all seems well.") + + +class OpenStackUpgradeTestsFocalUssuri(OpenStackUpgradeVMLaunchBase): + """Upgrade OpenStack from distro -> cloud:focal-victoria.""" + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do.""" + openstack_upgrade.run_upgrade_tests("cloud:focal-victoria") + + +class OpenStackUpgradeTests(OpenStackUpgradeVMLaunchBase): + """A Principal Class to encapsulate OpenStack Upgrade Tests. + + A generic Test class that can discover which Ubuntu version and OpenStack + version to upgrade from. + + TODO: Not used at present. Use the declarative tests directly that choose + the version to upgrade to. The functions that this class depends on need a + bit more work regarding how the determine which version to go to. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do. + + TODO: This is really inefficient at the moment, and doesn't (yet) + determine which ubuntu version to work from. Don't use until we can + make it better. + """ + # TODO: work out the most recent Ubuntu version; we assume this is the + # version that OpenStack is running on. + ubuntu_version = "focal" + logging.info("Getting all principle applications ...") + principle_services = upgrade_utils.get_all_principal_applications() + logging.info( + "Getting OpenStack vesions from principal applications ...") + current_versions = openstack_utils.get_current_os_versions( + principle_services) + logging.info("current versions: %s" % current_versions) + # Find the lowest value openstack release across all services and make + # sure all servcies are upgraded to one release higher than the lowest + from_version = upgrade_utils.get_lowest_openstack_version( + current_versions) + logging.info("from version: %s" % from_version) + to_version = upgrade_utils.determine_next_openstack_release( + from_version)[1] + logging.info("to version: %s" % to_version) + # TODO: need to determine the ubuntu base verion that is being upgraded + target_source = upgrade_utils.determine_new_source( + ubuntu_version, from_version, to_version, single_increment=True) + logging.info("target source: %s" % target_source) + assert target_source is not None + openstack_upgrade.run_upgrade_tests(target_source) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index d12911e91..bfa0f21b3 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -15,11 +15,14 @@ """Encapsulate OVN testing.""" import logging -import tenacity import juju +import tenacity + import zaza + +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -36,10 +39,19 @@ def setUpClass(cls): """Run class setup for OVN charm operation tests.""" super(BaseCharmOperationTest, cls).setUpClass() cls.services = ['NotImplemented'] # This must be overridden + cls.nrpe_checks = ['NotImplemented'] # This must be overridden cls.current_release = openstack_utils.get_os_release( openstack_utils.get_current_os_release_pair( cls.release_application or cls.application_name)) + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + # sleep for 2mins to allow 1min cron job to run... + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + def test_pause_resume(self): """Run pause and resume tests. @@ -50,6 +62,20 @@ def test_pause_resume(self): logging.info('Testing pause resume (services="{}")' .format(self.services)) + def test_nrpe_configured(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units(self.application_name) + cmds = [] + for check_name in self.nrpe_checks: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + if ret: + logging.info(ret) + self.assertIsNone(ret, msg=ret) + class CentralCharmOperationTest(BaseCharmOperationTest): """OVN Central Charm operation tests.""" @@ -62,6 +88,22 @@ def setUpClass(cls): 'ovn-northd', 'ovsdb-server', ] + source = zaza.model.get_application_config( + cls.application_name)['source']['value'] + logging.info(source) + if 'train' in source: + cls.nrpe_checks = [ + 'ovn-northd', + 'ovn-nb-ovsdb', + 'ovn-sb-ovsdb', + ] + else: + # Ussuri or later (distro or cloudarchive) + cls.nrpe_checks = [ + 'ovn-northd', + 'ovn-ovsdb-server-sb', + 'ovn-ovsdb-server-nb', + ] class ChassisCharmOperationTest(BaseCharmOperationTest): @@ -76,11 +118,38 @@ def setUpClass(cls): cls.services = [ 'ovn-controller', ] + if cls.application_name == 'ovn-chassis': + principal_app_name = 'magpie' + else: + principal_app_name = cls.application_name + source = zaza.model.get_application_config( + principal_app_name)['source']['value'] + logging.info(source) + if 'train' in source: + cls.nrpe_checks = [ + 'ovn-host', + 'ovs-vswitchd', + 'ovsdb-server', + ] + else: + # Ussuri or later (distro or cloudarchive) + cls.nrpe_checks = [ + 'ovn-controller', + 'ovsdb-server', + 'ovs-vswitchd', + ] class OVSOVNMigrationTest(test_utils.BaseCharmTest): """OVS to OVN migration tests.""" + @classmethod + def setUpClass(cls): + """Run class setup for OVN migration tests.""" + super(OVSOVNMigrationTest, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release( + openstack_utils.get_current_os_release_pair()) + def setUp(self): """Perform migration steps prior to validation.""" super(OVSOVNMigrationTest, self).setUp() @@ -348,3 +417,93 @@ def test_ovs_ovn_migration(self): zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states( states=self.target_deploy_status) + # Workaround for our old friend LP: #1852221 which hit us again on + # Groovy. We make the os_release check explicit so that we can + # re-evaluate the need for the workaround at the next release. + if self.current_release == openstack_utils.get_os_release( + 'groovy_victoria'): + try: + for application in ('ovn-chassis', 'ovn-dedicated-chassis'): + for unit in zaza.model.get_units(application): + zaza.model.run_on_unit( + unit.entity_id, + 'systemctl restart ovs-vswitchd') + except KeyError: + # One of the applications is not in the model, which is fine + pass + + +class OVNChassisDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-chassis') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('configure_ovs') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + return 'enable-sriov', str(not app_config['enable-sriov']['value']) + + +class OVNDedicatedChassisDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-dedicated-chassis') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('configure_ovs') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + new_value = str(not app_config['disable-mlockall'].get('value', False)) + return 'disable-mlockall', new_value + + +class OVNCentralDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + """Deferred restart tests for OVN Central.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-central') + + def run_tests(self): + """Run deferred restart tests.""" + # Charm does not defer hooks so that test is not included. + # Trigger a package change which requires a restart + self.run_package_change_test( + 'ovn-central', + 'ovn-central') diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index eca316dc9..fde38669b 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -337,8 +337,7 @@ def _get_keystone_session(self, ip, openrc, scope='DOMAIN'): logging.info('Authentication for {} on keystone IP {}' .format(openrc['OS_USERNAME'], ip)) if self.tls_rid: - openrc['OS_CACERT'] = \ - openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) @@ -402,9 +401,10 @@ def get_keystone_session_admin_user(self, ip): def test_003_test_override_is_observed(self): """Test that the override is observed by the underlying service.""" if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('groovy_victoria')): + openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( - "Test skipped until Bug #1880959 is fix released") + "Test skipped because bug #1880959 won't be fixed for " + "releases older than Queens") if self._test_name is None: logging.info("Doing policyd override for {}" .format(self._service_name)) @@ -572,6 +572,13 @@ def setUpClass(cls, application_name=None): super(GlanceTests, cls).setUpClass(application_name="glance") cls.application_name = "glance" + # NOTE(lourot): Same as NeutronApiTests. There is a race between the glance + # charm signalling its readiness and the service actually being ready to + # serve requests. The test will fail intermittently unless we gracefully + # accept this. + # Issue: openstack-charmers/zaza-openstack-tests#578 + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(8)) def get_client_and_attempt_operation(self, ip): """Attempt to list the images as a policyd override. @@ -681,5 +688,6 @@ def get_client_and_attempt_operation(self, ip): self.get_keystone_session_admin_user(ip)) try: octavia_client.provider_list() - except octaviaclient.OctaviaClientException: + except (octaviaclient.OctaviaClientException, + keystoneauth1.exceptions.http.Forbidden): raise PolicydOperationFailedException() diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d4a31115b..fed72df9f 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -428,3 +428,58 @@ def check_units(units): check_units(all_units) logging.info('OK') + + +class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='rabbitmq-server') + + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + pattern = '(Unit is ready|Unit is ready and clustered)$' + for unit in zaza.model.get_units(self.application_name): + zaza.model.block_until_unit_wl_message_match( + unit.entity_id, + pattern) + zaza.model.block_until_all_units_idle() + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + new_value = str(int( + app_config['connection-backlog'].get('value', 100) + 1)) + return 'connection-backlog', new_value + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'rabbitmq-server', + 'rabbitmq-server') + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in zaza.model.get_units(self.application_name): + zaza.model.run_action( + unit.entity_id, + 'restart-services', + action_params={'services': 'rabbitmq-server'}) + + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() diff --git a/zaza/openstack/charm_tests/saml_mellon/setup.py b/zaza/openstack/charm_tests/saml_mellon/setup.py index 7e5f96cc3..20b8cbdf1 100644 --- a/zaza/openstack/charm_tests/saml_mellon/setup.py +++ b/zaza/openstack/charm_tests/saml_mellon/setup.py @@ -25,6 +25,7 @@ cert as cert_utils, cli as cli_utils, openstack as openstack_utils, + generic as generic_utils, ) @@ -34,8 +35,8 @@ FEDERATED_GROUP = "federated_users" MEMBER = "Member" IDP = "samltest" +LOCAL_IDP_REMOTE_ID = "http://{}/simplesaml/saml2/idp/metadata.php" REMOTE_ID = "https://samltest.id/saml/idp" -MAP_NAME = "{}_mapping".format(IDP) PROTOCOL_NAME = "mapped" MAP_TEMPLATE = ''' [{{ @@ -45,7 +46,7 @@ "name": "{{0}}" }}, "group": {{ - "name": "federated_users", + "name": "{group_id}", "domain": {{ "id": "{domain_id}" }} @@ -55,7 +56,7 @@ "name": "{{0}}_project", "roles": [ {{ - "name": "Member" + "name": "{role_name}" }} ] }} @@ -81,7 +82,10 @@ ''' -def keystone_federation_setup(): +def keystone_federation_setup(federated_domain=FEDERATED_DOMAIN, + federated_group=FEDERATED_GROUP, + idp_name=IDP, + idp_remote_id=REMOTE_ID): """Configure Keystone Federation.""" cli_utils.setup_logging() keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -89,19 +93,19 @@ def keystone_federation_setup(): keystone_session) try: - domain = keystone_client.domains.find(name=FEDERATED_DOMAIN) + domain = keystone_client.domains.find(name=federated_domain) except keystoneauth1.exceptions.http.NotFound: domain = keystone_client.domains.create( - FEDERATED_DOMAIN, + federated_domain, description="Federated Domain", enabled=True) try: group = keystone_client.groups.find( - name=FEDERATED_GROUP, domain=domain) + name=federated_group, domain=domain) except keystoneauth1.exceptions.http.NotFound: group = keystone_client.groups.create( - FEDERATED_GROUP, + federated_group, domain=domain, enabled=True) @@ -109,30 +113,33 @@ def keystone_federation_setup(): keystone_client.roles.grant(role, group=group, domain=domain) try: - idp = keystone_client.federation.identity_providers.find( - name=IDP, domain_id=domain.id) + idp = keystone_client.federation.identity_providers.get(idp_name) except keystoneauth1.exceptions.http.NotFound: idp = keystone_client.federation.identity_providers.create( - IDP, - remote_ids=[REMOTE_ID], + idp_name, + remote_ids=[idp_remote_id], domain_id=domain.id, enabled=True) - JSON_RULES = json.loads(MAP_TEMPLATE.format(domain_id=domain.id)) + JSON_RULES = json.loads(MAP_TEMPLATE.format( + domain_id=domain.id, group_id=group.id, role_name=MEMBER)) + map_name = "{}_mapping".format(idp_name) try: - keystone_client.federation.mappings.find(name=MAP_NAME) + keystone_client.federation.mappings.get(map_name) except keystoneauth1.exceptions.http.NotFound: keystone_client.federation.mappings.create( - MAP_NAME, rules=JSON_RULES) + map_name, rules=JSON_RULES) try: - keystone_client.federation.protocols.get(IDP, PROTOCOL_NAME) + keystone_client.federation.protocols.get(idp_name, PROTOCOL_NAME) except keystoneauth1.exceptions.http.NotFound: keystone_client.federation.protocols.create( - PROTOCOL_NAME, mapping=MAP_NAME, identity_provider=idp) + PROTOCOL_NAME, mapping=map_name, identity_provider=idp) +# This setup method is deprecated. It will be removed once we fully drop the +# `samltest.id` dependency. def attach_saml_resources(application="keystone-saml-mellon"): """Attach resource to the Keystone SAML Mellon charm.""" test_idp_metadata_xml = "samltest.xml" @@ -161,3 +168,83 @@ def attach_saml_resources(application="keystone-saml-mellon"): fp.flush() zaza.model.attach_resource( application, sp_signing_keyinfo_name, fp.name) + + +def _attach_saml_resources_local_idp(keystone_saml_mellon_app_name=None, + test_saml_idp_app_name=None): + """Attach resources to the Keystone SAML Mellon and the local IdP.""" + action_result = zaza.model.run_action_on_leader( + test_saml_idp_app_name, 'get-idp-metadata') + idp_metadata = action_result.data['results']['output'] + + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'idp-metadata', + idp_metadata, + '.xml') + + (key, cert) = cert_utils.generate_cert('SP Signing Key') + + cert = cert.decode().strip("-----BEGIN CERTIFICATE-----") + cert = cert.strip("-----END CERTIFICATE-----") + + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'sp-private-key', + key.decode(), + '.pem') + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'sp-signing-keyinfo', + SP_SIGNING_KEY_INFO_XML_TEMPLATE.format(cert), + '.xml') + + action_result = zaza.model.run_action_on_leader( + keystone_saml_mellon_app_name, 'get-sp-metadata') + sp_metadata = action_result.data['results']['output'] + + generic_utils.attach_file_resource( + test_saml_idp_app_name, + 'sp-metadata', + sp_metadata, + '.xml') + + +def attach_saml_resources_idp1(): + """Attach the SAML resources for the local IdP #1.""" + _attach_saml_resources_local_idp( + keystone_saml_mellon_app_name="keystone-saml-mellon1", + test_saml_idp_app_name="test-saml-idp1") + + +def attach_saml_resources_idp2(): + """Attach the SAML resources for the local IdP #2.""" + _attach_saml_resources_local_idp( + keystone_saml_mellon_app_name="keystone-saml-mellon2", + test_saml_idp_app_name="test-saml-idp2") + + +def keystone_federation_setup_idp1(): + """Configure Keystone Federation for the local IdP #1.""" + test_saml_idp_unit = zaza.model.get_units("test-saml-idp1")[0] + idp_remote_id = LOCAL_IDP_REMOTE_ID.format( + test_saml_idp_unit.public_address) + + keystone_federation_setup( + federated_domain="federated_domain_idp1", + federated_group="federated_users_idp1", + idp_name="test-saml-idp1", + idp_remote_id=idp_remote_id) + + +def keystone_federation_setup_idp2(): + """Configure Keystone Federation for the local IdP #2.""" + test_saml_idp_unit = zaza.model.get_units("test-saml-idp2")[0] + idp_remote_id = LOCAL_IDP_REMOTE_ID.format( + test_saml_idp_unit.public_address) + + keystone_federation_setup( + federated_domain="federated_domain_idp2", + federated_group="federated_users_idp2", + idp_name="test-saml-idp2", + idp_remote_id=idp_remote_id) diff --git a/zaza/openstack/charm_tests/saml_mellon/tests.py b/zaza/openstack/charm_tests/saml_mellon/tests.py index 9b5b21146..962fa92c3 100644 --- a/zaza/openstack/charm_tests/saml_mellon/tests.py +++ b/zaza/openstack/charm_tests/saml_mellon/tests.py @@ -30,6 +30,8 @@ class FailedToReachIDP(Exception): pass +# This testing class is deprecated. It will be removed once we fully drop the +# `samltest.id` dependency. class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): """Charm Keystone SAML Mellon tests.""" @@ -156,3 +158,200 @@ def _do_redirect_check(url, region, idp_expect, horizon_expect): # We may need to try/except to allow horizon to build its pages _do_redirect_check(url, region, idp_expect, horizon_expect) logging.info("SUCCESS") + + +class BaseCharmKeystoneSAMLMellonTest(BaseKeystoneTest): + """Charm Keystone SAML Mellon tests.""" + + @classmethod + def setUpClass(cls, + application_name="keystone-saml-mellon", + test_saml_idp_app_name="test-saml-idp", + horizon_idp_option_name="myidp_mapped", + horizon_idp_display_name="myidp via mapped"): + """Run class setup for running Keystone SAML Mellon charm tests.""" + super(BaseCharmKeystoneSAMLMellonTest, cls).setUpClass() + # Note: The BaseKeystoneTest class sets the application_name to + # "keystone" which breaks keystone-saml-mellon actions. Explicitly set + # application name here. + cls.application_name = application_name + cls.test_saml_idp_app_name = test_saml_idp_app_name + cls.horizon_idp_option_name = horizon_idp_option_name + cls.horizon_idp_display_name = horizon_idp_display_name + cls.action = "get-sp-metadata" + cls.current_release = openstack_utils.get_os_release() + cls.FOCAL_USSURI = openstack_utils.get_os_release("focal_ussuri") + + @staticmethod + def check_horizon_redirect(horizon_url, horizon_expect, + horizon_idp_option_name, horizon_region, + idp_url, idp_expect): + """Validate the Horizon -> Keystone -> IDP redirects. + + This validation is done through `requests.session()`, and the proper + get / post http calls. + + :param horizon_url: The login page for the Horizon OpenStack dashboard. + :type horizon_url: string + :param horizon_expect: Information that needs to be displayed by + Horizon login page, when there is a proper + SAML IdP configuration. + :type horizon_expect: string + :param horizon_idp_option_name: The name of the IdP that is chosen + in the Horizon dropdown from the login + screen. This will go in the post body + as 'auth_type'. + :type horizon_idp_option_name: string + :param horizon_region: Information needed to complete the http post + data for the Horizon login. + :type horizon_region: string + :param idp_url: The url for the IdP where the user needs to be + redirected. + :type idp_url: string + :param idp_expect: Information that needs to be displayed by the IdP + after the user is redirected there. + :type idp_expect: string + :returns: None + """ + # start session, get csrftoken + client = requests.session() + # Verify=False see note below + login_page = client.get(horizon_url, verify=False) + + # Validate SAML method is available + assert horizon_expect in login_page.text + + # Get cookie + if "csrftoken" in client.cookies: + csrftoken = client.cookies["csrftoken"] + else: + raise Exception("Missing csrftoken") + + # Build and send post request + form_data = { + "auth_type": horizon_idp_option_name, + "csrfmiddlewaretoken": csrftoken, + "next": "/horizon/project/api_access", + "region": horizon_region, + } + + # Verify=False due to CA certificate bundles. + # If we don't set it validation fails for keystone/horizon + # We would have to install the keystone CA onto the system + # to validate end to end. + response = client.post( + horizon_url, + data=form_data, + headers={"Referer": horizon_url}, + allow_redirects=True, + verify=False) + + if idp_expect not in response.text: + msg = "FAILURE code={} text={}".format(response, response.text) + # Raise a custom exception. + raise FailedToReachIDP(msg) + + # Validate that we were redirected to the proper IdP + assert response.url.startswith(idp_url) + assert idp_url in response.text + + def test_run_get_sp_metadata_action(self): + """Validate the get-sp-metadata action.""" + unit = zaza.model.get_units(self.application_name)[0] + ip = self.vip if self.vip else unit.public_address + + action = zaza.model.run_action(unit.entity_id, self.action) + self.assertNotIn( + "failed", + action.data["status"], + msg="The action failed: {}".format(action.data["message"])) + + output = action.data["results"]["output"] + root = etree.fromstring(output) + for item in root.items(): + if "entityID" in item[0]: + self.assertIn(ip, item[1]) + + for appt in root.getchildren(): + for elem in appt.getchildren(): + for item in elem.items(): + if "Location" in item[0]: + self.assertIn(ip, item[1]) + + logging.info("Successul get-sp-metadata action") + + def test_saml_mellon_redirects(self): + """Validate the horizon -> keystone -> IDP redirects.""" + unit = zaza.model.get_units(self.application_name)[0] + keystone_ip = self.vip if self.vip else unit.public_address + + horizon = "openstack-dashboard" + horizon_config = zaza.model.get_application_config(horizon) + horizon_vip = horizon_config.get("vip").get("value") + unit = zaza.model.get_units("openstack-dashboard")[0] + + horizon_ip = horizon_vip if horizon_vip else unit.public_address + proto = "https" if self.tls_rid else "http" + + # Use Keystone URL for < Focal + if self.current_release < self.FOCAL_USSURI: + region = "{}://{}:5000/v3".format(proto, keystone_ip) + else: + region = "default" + + idp_address = zaza.model.get_units( + self.test_saml_idp_app_name)[0].public_address + + horizon_url = "{}://{}/horizon/auth/login/".format(proto, horizon_ip) + horizon_expect = ''.format( + self.horizon_idp_option_name, self.horizon_idp_display_name) + idp_url = ("http://{0}/simplesaml/" + "module.php/core/loginuserpass.php").format(idp_address) + # This is the message the local test-saml-idp displays after you are + # redirected. It shows we have been directed to: + # horizon -> keystone -> test-saml-idp + idp_expect = ( + "A service has requested you to authenticate yourself. Please " + "enter your username and password in the form below.") + + # Execute the check + BaseCharmKeystoneSAMLMellonTest.check_horizon_redirect( + horizon_url=horizon_url, + horizon_expect=horizon_expect, + horizon_idp_option_name=self.horizon_idp_option_name, + horizon_region=region, + idp_url=idp_url, + idp_expect=idp_expect) + logging.info("SUCCESS") + + +class CharmKeystoneSAMLMellonIDP1Test(BaseCharmKeystoneSAMLMellonTest): + """Charm Keystone SAML Mellon tests class for the local IDP #1.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone SAML Mellon charm tests. + + It does the necessary setup for the local IDP #1. + """ + super(CharmKeystoneSAMLMellonIDP1Test, cls).setUpClass( + application_name="keystone-saml-mellon1", + test_saml_idp_app_name="test-saml-idp1", + horizon_idp_option_name="test-saml-idp1_mapped", + horizon_idp_display_name="Test SAML IDP #1") + + +class CharmKeystoneSAMLMellonIDP2Test(BaseCharmKeystoneSAMLMellonTest): + """Charm Keystone SAML Mellon tests class for the local IDP #2.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone SAML Mellon charm tests. + + It does the necessary setup for the local IDP #2. + """ + super(CharmKeystoneSAMLMellonIDP2Test, cls).setUpClass( + application_name="keystone-saml-mellon2", + test_saml_idp_app_name="test-saml-idp2", + horizon_idp_option_name="test-saml-idp2_mapped", + horizon_idp_display_name="Test SAML IDP #2") diff --git a/zaza/openstack/charm_tests/security/tests.py b/zaza/openstack/charm_tests/security/tests.py deleted file mode 100644 index 741b53ba0..000000000 --- a/zaza/openstack/charm_tests/security/tests.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Encapsulate general security testing.""" - -import unittest - -import zaza.model as model -import zaza.charm_lifecycle.utils as utils -from zaza.openstack.utilities.file_assertions import ( - assert_path_glob, - assert_single_file, -) - - -def _make_test_function(application, file_details, paths=None): - """Generate a test function given the specified inputs. - - :param application: Application name to assert file ownership on - :type application: str - :param file_details: Dictionary of file details to test - :type file_details: dict - :param paths: List of paths to test in this application - :type paths: Optional[list(str)] - :returns: Test function - :rtype: unittest.TestCase - """ - def test(self): - for unit in model.get_units(application): - unit = unit.entity_id - if '*' in file_details['path']: - assert_path_glob(self, unit, file_details, paths) - else: - assert_single_file(self, unit, file_details) - return test - - -def _add_tests(): - """Add tests to the unittest.TestCase.""" - def class_decorator(cls): - """Add tests based on input yaml to `cls`.""" - files = utils.get_charm_config('./file-assertions.yaml') - deployed_applications = model.sync_deployed() - for name, attributes in files.items(): - # Lets make sure to only add tests for deployed applications - if name in deployed_applications: - paths = [ - file['path'] for - file in attributes['files'] - if "*" not in file["path"] - ] - for file in attributes['files']: - test_func = _make_test_function(name, file, paths=paths) - setattr( - cls, - 'test_{}_{}'.format(name, file['path']), - test_func) - return cls - return class_decorator - - -class FileOwnershipTest(unittest.TestCase): - """Encapsulate File ownership tests.""" - - pass - - -FileOwnershipTest = _add_tests()(FileOwnershipTest) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index a49eae674..e524fa2ae 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -21,6 +21,7 @@ import os import sys import unittest +import juju from zaza import model from zaza.openstack.utilities import ( @@ -55,6 +56,12 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): @classmethod def setUpClass(cls): """Run setup for Series Upgrades.""" + # NOTE(ajkavanagh): Set the jujulib Connection frame size to 4GB to + # cope with all the outputs from series upgrade; long term, don't send + # that output back, which will require that the upgrade function in the + # charm doesn't capture the output of the upgrade in the action, but + # instead puts it somewhere that can by "juju scp"ed. + juju.client.connection.Connection.MAX_FRAME_SIZE = 2**32 cli_utils.setup_logging() cls.from_series = None cls.to_series = None @@ -89,19 +96,25 @@ def test_200_run_series_upgrade(self): upgrade_function = \ parallel_series_upgrade.parallel_series_upgrade + # allow up to 4 parallel upgrades at a time. This is to limit the + # amount of data/calls that asyncio is handling as it's gets + # unstable if all the applications are done at the same time. + sem = asyncio.Semaphore(4) for charm_name in apps: charm = applications[charm_name]['charm'] name = upgrade_utils.extract_charm_name_from_url(charm) upgrade_config = parallel_series_upgrade.app_config(name) upgrade_functions.append( - upgrade_function( - charm_name, - **upgrade_config, - from_series=from_series, - to_series=to_series, - completed_machines=completed_machines, - workaround_script=workaround_script, - files=files)) + wrap_coroutine_with_sem( + sem, + upgrade_function( + charm_name, + **upgrade_config, + from_series=from_series, + to_series=to_series, + completed_machines=completed_machines, + workaround_script=workaround_script, + files=files))) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_functions)) model.block_until_all_units_idle() @@ -109,6 +122,18 @@ def test_200_run_series_upgrade(self): logging.info("Done!") +async def wrap_coroutine_with_sem(sem, coroutine): + """Wrap a coroutine with a semaphore to limit concurrency. + + :param sem: The semaphore to limit concurrency + :type sem: asyncio.Semaphore + :param coroutine: the corouting to limit concurrency + :type coroutine: types.CoroutineType + """ + async with sem: + await coroutine + + class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest): """OpenStack Series Upgrade. diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 5c85d9c03..7ae0ba31d 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -17,6 +17,7 @@ import jinja2 import urllib.parse import os +import subprocess import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.juju as juju_utils @@ -170,6 +171,27 @@ def add_keystone_config(ctxt, keystone_session): ctxt['default_domain_id'] = domain.id +def add_octavia_config(ctxt): + """Add octavia config to context. + + :param ctxt: Context dictionary + :type ctxt: dict + :returns: None + :rtype: None + :raises: subprocess.CalledProcessError + """ + subprocess.check_call([ + 'curl', + "{}:80/swift/v1/fixtures/test_server.bin".format( + ctxt['test_swift_ip']), + '-o', "{}/test_server.bin".format(ctxt['workspace_path']) + ]) + subprocess.check_call([ + 'chmod', '+x', + "{}/test_server.bin".format(ctxt['workspace_path']) + ]) + + def get_service_list(keystone_session): """Retrieve list of services from keystone. @@ -227,7 +249,7 @@ def add_auth_config(ctxt): overcloud_auth['OS_PROJECT_DOMAIN_NAME']) -def get_tempest_context(): +def get_tempest_context(workspace_path): """Generate the tempest config context. :returns: Context dictionary @@ -235,6 +257,7 @@ def get_tempest_context(): """ keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} + ctxt['workspace_path'] = workspace_path ctxt_funcs = { 'nova': add_nova_config, 'neutron': add_neutron_config, @@ -253,6 +276,8 @@ def get_tempest_context(): ctxt_func(ctxt, keystone_session) add_environment_var_config(ctxt, ctxt['enabled_services']) add_auth_config(ctxt) + if 'octavia' in ctxt['enabled_services']: + add_octavia_config(ctxt) return ctxt @@ -289,13 +314,14 @@ def setup_tempest(tempest_template, accounts_template): workspace_name, workspace_path = tempest_utils.get_workspace() tempest_utils.destroy_workspace(workspace_name, workspace_path) tempest_utils.init_workspace(workspace_path) + context = get_tempest_context(workspace_path) render_tempest_config( os.path.join(workspace_path, 'etc/tempest.conf'), - get_tempest_context(), + context, tempest_template) render_tempest_config( os.path.join(workspace_path, 'etc/accounts.yaml'), - get_tempest_context(), + context, accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 index b441f2d86..5cb65edaf 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -4,7 +4,6 @@ use_stderr = false log_file = tempest.log [auth] -test_accounts_file = accounts.yaml default_credentials_domain_name = {{ default_credentials_domain_name }} admin_username = {{ admin_username }} admin_project_name = {{ admin_project_name }} @@ -65,6 +64,7 @@ floating_network_name = {{ ext_net }} [network-feature-enabled] ipv6 = false api_extensions = {{ neutron_api_extensions }} +port_security = true {% endif %} {% if 'heat' in enabled_services %} @@ -104,3 +104,10 @@ catalog_type = {{ catalog_type }} [volume-feature-enabled] backup = false {% endif %} + +{% if 'octavia' in enabled_services %} +[load_balancer] +enable_security_groups = true +test_with_ipv6 = false +test_server_path = {{ workspace_path }}/test_server.bin +{% endif %} \ No newline at end of file diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index a3614f02e..bc67933ef 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -18,6 +18,7 @@ import sys import tenacity import unittest +import yaml import novaclient @@ -120,22 +121,102 @@ def tearDown(self): @classmethod def setUpClass(cls, application_name=None, model_alias=None): - """Run setup for test class to create common resources.""" + """Run setup for test class to create common resources. + + Note: the derived class may not use the application_name; if it's set + to None then this setUpClass() method will attempt to extract the + application name from the charm_config (essentially the test.yaml) + using the key 'charm_name' in the test_config. If that isn't present, + then there will be no application_name set, and this is considered a + generic scenario of a whole model rather than a particular charm under + test. + + :param application_name: the name of the applications that the derived + class is testing. If None, then it's a generic test not connected + to any single charm. + :type application_name: Optional[str] + :param model_alias: the alias to use if needed. + :type model_alias: Optional[str] + """ cls.model_aliases = model.get_juju_model_aliases() if model_alias: cls.model_name = cls.model_aliases[model_alias] else: cls.model_name = model.get_juju_model() cls.test_config = lifecycle_utils.get_charm_config(fatal=False) + if application_name: cls.application_name = application_name else: - cls.application_name = cls.test_config['charm_name'] + try: + charm_under_test_name = cls.test_config['charm_name'] + except KeyError: + logging.warning("No application_name and no charm config so " + "not setting the application_name. Likely a " + "scenario test.") + return + deployed_app_names = model.sync_deployed(model_name=cls.model_name) + if charm_under_test_name in deployed_app_names: + # There is an application named like the charm under test. + # Let's consider it the application under test: + cls.application_name = charm_under_test_name + else: + # Let's search for any application whose name starts with the + # name of the charm under test and assume it's the application + # under test: + for app_name in deployed_app_names: + if app_name.startswith(charm_under_test_name): + cls.application_name = app_name + break + else: + logging.warning('Could not find application under test') + return + cls.lead_unit = model.get_lead_unit_name( cls.application_name, model_name=cls.model_name) logging.debug('Leader unit is {}'.format(cls.lead_unit)) + def config_current_separate_non_string_type_keys( + self, non_string_type_keys, config_keys=None, + application_name=None): + """Obtain current config and the non-string type config separately. + + If the charm config option is not string, it will not accept being + reverted back in "config_change()" method if the current value is None. + Therefore, obtain the current config and separate those out, so they + can be used for a separate invocation of "config_change()" with + reset_to_charm_default set to True. + + :param config_keys: iterable of strs to index into the current config. + If None, return all keys from the config + :type config_keys: Optional[Iterable[str]] + :param non_string_type_keys: list of non-string type keys to be + separated out only if their current value + is None + :type non_string_type_keys: list + :param application_name: String application name for use when called + by a charm under test other than the object's + application. + :type application_name: Optional[str] + :return: Dictionary of current charm configs without the + non-string type keys provided, and dictionary of the + non-string keys found in the supplied config_keys list. + :rtype: Dict[str, Any], Dict[str, None] + """ + current_config = self.config_current(application_name, config_keys) + non_string_type_config = {} + if config_keys is None: + config_keys = list(current_config.keys()) + for key in config_keys: + # We only care if the current value is None, otherwise it will + # not face issues being reverted by "config_change()" + if key in non_string_type_keys and current_config[key] is None: + non_string_type_config[key] = None + current_config.pop(key) + + return current_config, non_string_type_config + def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. @@ -257,7 +338,7 @@ def config_change(self, default_config, alternate_config, 'charm default: "{}"' .format(alternate_config.keys())) model.reset_application_config(application_name, - alternate_config.keys(), + list(alternate_config.keys()), model_name=self.model_name) elif default_config == alternate_config: logging.debug('default_config == alternate_config, not attempting ' @@ -492,6 +573,32 @@ def get_my_tests_options(self, key, default=None): return self.test_config.get('tests_options', {}).get( '.'.join(caller_path + [key]), default) + def get_applications_with_substring_in_name(self, substring): + """Get applications with substring in name. + + :param substring: String to search for in application names + :type substring: str + :returns: List of matching applictions + :rtype: List + """ + status = model.get_status().applications + applications = [] + for application in status.keys(): + if substring in application: + applications.append(application) + return applications + + def run_update_status_hooks(self, units): + """Run update status hooks on units. + + :param units: List of unit names or unit.entity_id + :type units: List[str] + :returns: None + :rtype: None + """ + for unit in units: + model.run_on_unit(unit, "hooks/update-status") + class OpenStackBaseTest(BaseCharmTest): """Generic helpers for testing OpenStack API charms.""" @@ -526,7 +633,8 @@ def resource_cleanup(self): # Test did not define self.RESOURCE_PREFIX, ignore. pass - def launch_guest(self, guest_name, userdata=None): + def launch_guest(self, guest_name, userdata=None, use_boot_volume=False, + instance_key=None): """Launch two guests to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -539,25 +647,38 @@ def launch_guest(self, guest_name, userdata=None): :type guest_name: str :param userdata: Userdata to attach to instance :type userdata: Optional[str] + :param use_boot_volume: Whether to boot guest from a shared volume. + :type use_boot_volume: boolean + :param instance_key: Key to collect associated config data with. + :type instance_key: Optional[str] :returns: Nova instance objects :rtype: Server """ + instance_key = instance_key or glance_setup.LTS_IMAGE_NAME instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name) - instance = self.retrieve_guest(instance_name) - if instance: - logging.info('Removing already existing instance ({}) with ' - 'requested name ({})' - .format(instance.id, instance_name)) - openstack_utils.delete_resource( - self.nova_client.servers, - instance.id, - msg="server") - - return configure_guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name=instance_name, - userdata=userdata) + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)): + with attempt: + old_instance_with_same_name = self.retrieve_guest( + instance_name) + if old_instance_with_same_name: + logging.info( + 'Removing already existing instance ({}) with ' + 'requested name ({})' + .format(old_instance_with_same_name.id, instance_name)) + openstack_utils.delete_resource( + self.nova_client.servers, + old_instance_with_same_name.id, + msg="server") + + return configure_guest.launch_instance( + instance_key, + vm_name=instance_name, + use_boot_volume=use_boot_volume, + userdata=userdata) def launch_guests(self, userdata=None): """Launch two guests to use in tests. @@ -572,15 +693,10 @@ def launch_guests(self, userdata=None): """ launched_instances = [] for guest_number in range(1, 2+1): - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=10)): - with attempt: - launched_instances.append( - self.launch_guest( - guest_name='ins-{}'.format(guest_number), - userdata=userdata)) + launched_instances.append( + self.launch_guest( + guest_name='ins-{}'.format(guest_number), + userdata=userdata)) return launched_instances def retrieve_guest(self, guest_name): @@ -612,3 +728,400 @@ def retrieve_guests(self): instance_2 = self.retrieve_guest( '{}-ins-1'.format(self.RESOURCE_PREFIX)) return instance_1, instance_2 + + +class BaseDeferredRestartTest(BaseCharmTest): + """Check deferred restarts. + + Example of adding a deferred restart test:: + + class NeutronOVSDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + + @classmethod + def setUpClass(cls): + super().setUpClass(application_name='neutron-openvswitch') + + def run_tests(self): + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + + NOTE: The test has been broken into various class methods which may require + specialisation if the charm being tested is not a standard OpenStack + charm e.g. `trigger_deferred_hook_via_charm` if the charm is not + an oslo config or does not have a debug option. + """ + + @classmethod + def setUpClass(cls, application_name): + """Run test setup. + + :param application_name: Name of application to run tests against. + :type application_name: str + """ + cls.application_name = application_name + super().setUpClass(application_name=cls.application_name) + + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + model.block_until_unit_wl_message_match( + unit.entity_id, + 'Unit is ready') + model.block_until_all_units_idle() + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + logging.info("Running restart-services on {}".format( + unit.entity_id)) + model.run_action( + unit.entity_id, + 'restart-services', + action_params={'deferred-only': True}, + raise_on_failure=True) + + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() + + def clear_hooks(self): + """Clear and deferred hooks. + + Run any deferred hooks. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + logging.info("Running run-deferred-hooks on {}".format( + unit.entity_id)) + model.run_action( + unit.entity_id, + 'run-deferred-hooks', + raise_on_failure=True) + + def check_clear_hooks(self): + """Clear deferred hooks and check status. + + Clear deferred hooks and then check the workload status message + for each unit. + """ + self.clear_hooks() + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() + + def run_show_deferred_events_action(self): + """Run show-deferred-events and return results. + + :returns: Data from action run + :rtype: Dict + """ + unit = model.get_units(self.application_name)[0] + action = model.run_action( + unit.entity_id, + 'show-deferred-events', + raise_on_failure=True) + return yaml.safe_load(action.data['results']['output']) + + def check_show_deferred_events_action_restart(self, test_service, + restart_reason): + """Check the output from the action to list deferred restarts. + + Run the action to list any deferred restarts and check it has entry for + the given service and reason. + + :param test_service: Service that should need a restart + :type test_service: str + :param restart_reason: The reason the action should list for the + service needing to be restarted. This can be a + substring. + :type restart_reason: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-events action").format( + test_service)) + for event in self.run_show_deferred_events_action()['restarts']: + logging.info("{} in {} and {} in {}".format( + test_service, + event, + restart_reason, + event)) + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_events_action_hook(self, hook): + """Check the output from the action to list deferred eveents. + + Run the action to list any deferred events and check it has entry for + the given hook. + + :param hook: Hook or method name + :type hook: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as skipped in " + "show-deferred-events action").format(hook)) + for event in self.run_show_deferred_events_action()['hooks']: + logging.info("{} in {}".format(hook, event)) + if hook in event: + break + else: + msg = '{} not found in {}'.format(hook, event) + raise Exception(msg) + + def check_show_deferred_restarts_wlm(self, test_service): + """Check the workload status message lists deferred restart. + + :param test_service: Service that should need a restart + :type test_service: str + """ + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + # Just checking one example service should we be checking all? + logging.info( + ("Checking {} is marked as needing restart in workload " + "message of {}".format(test_service, unit.entity_id))) + assert test_service in unit.workload_status_message + + def check_deferred_hook_wlm(self, deferred_hook): + """Check the workload status message lists deferred event. + + :param deferred_hook: Hook or method name which should be showing as + deferred. + :type deferred_hook: str + """ + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + logging.info( + ("Checking {} is marked as having deferred hook in workload " + "message".format(unit.entity_id))) + assert deferred_hook in unit.workload_status_message + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + NOTE: The implementation assumes the charm has a `debug` option and + If that is not true the derived class should override this + method. + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = model.get_application_config(self.application_name) + return 'debug', str(not app_config['debug']['value']) + + def set_new_config(self): + """Change applications charm config.""" + logging.info("Triggering deferred restart via config change") + config_key, new_value = self.get_new_config() + logging.info("Setting {}: {}".format(config_key, new_value)) + model.set_application_config( + self.application_name, + {config_key: new_value}) + return new_value + + def trigger_deferred_restart_via_charm(self, restart_config_file): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the restart_config_file in an oslo + config file. If that is not true the derived class should + override this method. + + :param restart_config_file: Config file that updated value is expected + in. + :type restart_config_file: str + """ + new_debug_value = self.set_new_config() + expected_contents = { + 'DEFAULT': { + 'debug': [new_debug_value]}} + logging.info("Waiting for debug to be {} in {}".format( + new_debug_value, + restart_config_file)) + model.block_until_oslo_config_entries_match( + self.application_name, + restart_config_file, + expected_contents) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + + def trigger_deferred_hook_via_charm(self, deferred_hook): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be rendered in + applications config file. + + :param deferred_hook: Hook or method name which should be showing as + deferred. + :type deferred_hook: str + :returns: New config value + :rtype: Union[str, int, float] + """ + new_debug_value = self.set_new_config() + for unit in model.get_units(self.application_name): + logging.info('Waiting for {} to show deferred hook'.format( + unit.entity_id)) + model.block_until_unit_wl_message_match( + unit.entity_id, + status_pattern='.*{}.*'.format(deferred_hook)) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + return new_debug_value + + def trigger_deferred_restart_via_package(self, restart_package): + """Update a package which requires a service restart. + + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + """ + logging.info("Triggering deferred restart via package change") + # Test restart requested by package + for unit in model.get_units(self.application_name): + model.run_on_unit( + unit.entity_id, + ('dpkg-reconfigure {}; ' + 'JUJU_HOOK_NAME=update-status ./hooks/update-status').format( + restart_package)) + + def run_charm_change_restart_test(self, test_service, restart_config_file): + """Trigger a deferred restart by updating a config file via the charm. + + Trigger a hook in the charm which the charm will defer. + + :param test_service: Service that should need a restart + :type test_service: str + :param restart_config_file: Config file that updated value is expected + in. + :type restart_config_file: str + """ + self.trigger_deferred_restart_via_charm(restart_config_file) + + self.check_show_deferred_restarts_wlm(test_service) + self.check_show_deferred_events_action_restart( + test_service, + restart_config_file) + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_charm_change_hook_test(self, deferred_hook): + """Trigger a deferred restart by updating a config file via the charm. + + :param deferred_hook: Hook or method name which should be showing as + defeerred. + :type deferred_hook: str + """ + self.trigger_deferred_hook_via_charm(deferred_hook) + + self.check_deferred_hook_wlm(deferred_hook) + self.check_show_deferred_events_action_hook(deferred_hook) + # Rerunning to flip config option back to previous value. + self.trigger_deferred_hook_via_charm(deferred_hook) + logging.info("Running restart action to clear deferred hooks") + # If there are a number of units in the application and restarts take + # time then another deferred hook can occur so do not block on a + # clear status message. + self.clear_hooks() + + def run_package_change_test(self, restart_package, restart_package_svc): + """Trigger a deferred restart by updating a package. + + Update a package which requires will add a deferred restart. + + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str + """ + self.trigger_deferred_restart_via_package(restart_package) + + self.check_show_deferred_restarts_wlm(restart_package_svc) + self.check_show_deferred_events_action_restart( + restart_package_svc, + 'Package update') + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_tests(self): + """Run charm tests. should specify which tests to run. + + The charm test that implements this test should specify which tests to + run, for example: + + def run_tests(self): + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a config change which requires a restart + self.run_charm_change_restart_test( + 'neutron-l3-agent', + '/etc/neutron/neutron.conf') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + """ + raise NotImplementedError + + def test_deferred_restarts(self): + """Run deferred restart tests.""" + app_config = model.get_application_config(self.application_name) + auto_restart_config_key = 'enable-auto-restarts' + if auto_restart_config_key not in app_config: + raise unittest.SkipTest("Deferred restarts not implemented") + + # Ensure auto restarts are off. + policy_file = '/etc/policy-rc.d/charm-{}.policy'.format( + self.application_name) + if app_config[auto_restart_config_key]['value']: + logging.info("Turning off auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'False'}) + logging.info("Waiting for {} to appear on units of {}".format( + policy_file, + self.application_name)) + model.block_until_file_has_contents( + self.application_name, + policy_file, + 'policy_requestor_name') + # The block_until_file_has_contents ensures the change we waiting + # for has happened, now just wait for any hooks to finish. + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + else: + logging.info("Auto restarts already disabled") + + self.run_tests() + + # Finished so turn auto-restarts back on. + logging.info("Turning on auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'True'}) + model.block_until_file_missing( + self.application_name, + policy_file) + model.block_until_all_units_idle() + self.check_clear_hooks() diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index a7ba7b329..11fe187e7 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -19,12 +19,16 @@ import logging import os +import boto3 + +import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model as zaza_model import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.openstack as openstack_utils -def basic_setup(): +def nfs_setup(): """Run setup for testing Trilio. Setup for testing Trilio is currently part of functional @@ -35,12 +39,14 @@ def basic_setup(): trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") nfs_shares_conf = {"nfs-shares": "{}:/srv/testing".format(nfs_server_ip)} + logging.info("NFS share config: {}".format(nfs_shares_conf)) _trilio_services = ["trilio-wlm", "trilio-data-mover"] conf_changed = False for juju_service in _trilio_services: app_config = zaza_model.get_application_config(juju_service) if app_config["nfs-shares"] != nfs_shares_conf["nfs-shares"]: + logging.info("Updating nfs-shares config option") zaza_model.set_application_config(juju_service, nfs_shares_conf) conf_changed = True @@ -54,6 +60,9 @@ def basic_setup(): target_status="active", ) + +def trust_setup(): + """Run setup Trilio trust setup.""" logging.info("Executing create-cloud-admin-trust") password = juju_utils.leader_get("keystone", "admin_passwd") @@ -66,6 +75,9 @@ def basic_setup(): ) ) + +def license_setup(): + """Run setup Trilio license setup.""" logging.info("Executing create-license") test_license = os.environ.get("TEST_TRILIO_LICENSE") if test_license and os.path.exists(test_license): @@ -81,3 +93,86 @@ def basic_setup(): else: logging.error("Unable to find Trilio License file") + + +def s3_setup(): + """Run setup of s3 options for Trilio.""" + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client( + session) + + # Get token data so we can glean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + s3_region = endpoint['region'] + s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + ec2_creds = ks_client.ec2.create(user_id, project_id) + cacert = openstack_utils.get_cacert() + kwargs = { + 'region_name': s3_region, + 'aws_access_key_id': ec2_creds.access, + 'aws_secret_access_key': ec2_creds.secret, + 'endpoint_url': s3_endpoint, + 'verify': cacert, + } + s3 = boto3.resource('s3', **kwargs) + + # Create bucket + bucket_name = 'zaza-trilio' + logging.info("Creating bucket: {}".format(bucket_name)) + bucket = s3.Bucket(bucket_name) + bucket.create() + + s3_config = { + 'tv-s3-secret-key': ec2_creds.secret, + 'tv-s3-access-key': ec2_creds.access, + 'tv-s3-region-name': s3_region, + 'tv-s3-bucket': bucket_name, + 'tv-s3-endpoint-url': s3_endpoint} + for app in ['trilio-wlm', 'trilio-data-mover']: + logging.info("Setting s3 config for {}".format(app)) + zaza_model.set_application_config(app, s3_config) + test_config = lifecycle_utils.get_charm_config(fatal=False) + states = test_config.get('target_deploy_status', {}) + states['trilio-wlm'] = { + 'workload-status': 'blocked', + 'workload-status-message': 'application not trusted'} + zaza_model.wait_for_application_states( + states=test_config.get('target_deploy_status', {}), + timeout=7200) + zaza_model.block_until_all_units_idle() + + +def basic_setup(): + """Run basic setup for Trilio apps.""" + backup_target_type = zaza_model.get_application_config( + 'trilio-wlm')['backup-target-type']['value'] + if backup_target_type == "nfs": + nfs_setup() + if backup_target_type in ["s3", "experimental-s3"]: + s3_setup() + trust_setup() + license_setup() + + +def python2_workaround(): + """Workaround for Bug #1915914. + + Trilio code currently has a bug which assumes an executable called 'python' + will be on the path. To workaround this install a package which adds a + symlink to python + """ + for unit in zaza_model.get_units('trilio-wlm'): + zaza_model.run_on_unit( + unit.entity_id, + ("apt install --yes python-is-python3; " + "systemctl restart wlm\\*.service")) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 581c395f2..ad477d83f 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -262,7 +262,7 @@ def create_snapshot(self, workload_id): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=30), - stop=tenacity.stop_after_delay(720), + stop=tenacity.stop_after_delay(900), reraise=True, ) @@ -440,7 +440,7 @@ def test_ghost_nfs_share(self): ) -class TrilioWLMTest(TrilioGhostNFSShareTest): +class TrilioWLMBaseTest(TrilioBaseTest): """Tests for Trilio Workload Manager charm.""" conf_file = "/etc/workloadmgr/workloadmgr.conf" @@ -463,10 +463,26 @@ class TrilioDMAPITest(TrilioBaseTest): services = ["dmapi-api"] -class TrilioDataMoverTest(TrilioGhostNFSShareTest): +class TrilioDataMoverBaseTest(TrilioBaseTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" application_name = "trilio-data-mover" services = ["tvault-contego"] + + +class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): + """Tests for Trilio Data Mover charm backed by NFS.""" + + +class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): + """Tests for Trilio Data Mover charm backed by S3.""" + + +class TrilioWLMNFSTest(TrilioWLMBaseTest, TrilioGhostNFSShareTest): + """Tests for Trilio WLM charm backed by NFS.""" + + +class TrilioWLMS3Test(TrilioWLMBaseTest): + """Tests for Trilio WLM charm backed by S3.""" diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 4db90b52e..c792508fc 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -26,6 +26,7 @@ import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.openstack.utilities.generic +import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.utilities.juju as juju_utils @@ -73,9 +74,27 @@ def basic_setup_and_unseal(cacert=None): zaza.model.run_on_unit(unit.name, './hooks/update-status') +async def mojo_or_default_unseal_by_unit(): + """Unseal any units reported as sealed using a cacert. + + The mojo cacert is tried first, and if that doesn't exist, then the default + zaza located cacert is used. + """ + try: + await mojo_unseal_by_unit() + except zaza_exceptions.CACERTNotFound: + await unseal_by_unit() + + def mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + unseal_by_unit(cacert) + + +def unseal_by_unit(cacert=None): + """Unseal any units reported as sealed using mojo cacert.""" + cacert = cacert or get_cacert_file() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): @@ -86,9 +105,27 @@ def mojo_unseal_by_unit(): zaza.model.run_on_unit(unit_name, './hooks/update-status') +async def async_mojo_or_default_unseal_by_unit(): + """Unseal any units reported as sealed using a cacert. + + The mojo cacert is tried first, and if that doesn't exist, then the default + zaza located cacert is used. + """ + try: + await async_mojo_unseal_by_unit() + except zaza_exceptions.CACERTNotFound: + await async_unseal_by_unit() + + async def async_mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + await async_unseal_by_unit(cacert) + + +async def async_unseal_by_unit(cacert=None): + """Unseal any units reported as sealed using vault cacert.""" + cacert = cacert or get_cacert_file() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): @@ -137,7 +174,8 @@ def auto_initialize(cacert=None, validation_application='keystone', wait=True): zaza.model.wait_for_agent_status() test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + states=test_config.get('target_deploy_status', {}), + timeout=7200) if validation_application: validate_ca(cacertificate, application=validation_application) @@ -184,9 +222,8 @@ def validate_ca(cacertificate, application="keystone", port=5000): :returns: None :rtype: None """ - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( application, - zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, cacertificate.decode().strip()) vip = (zaza.model.get_application_config(application) .get("vip").get("value")) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 40227fd06..9b898bff8 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -18,11 +18,13 @@ import contextlib import hvac +import json import logging import time import unittest import uuid import tempfile +import tenacity import requests import zaza.charm_lifecycle.utils as lifecycle_utils @@ -31,6 +33,7 @@ import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.model +import zaza.utilities.juju as juju_utils class BaseVaultTest(test_utils.OpenStackBaseTest): @@ -110,7 +113,11 @@ def test_unseal(self, test_config=None): vault_utils.run_charm_authorize(self.vault_creds['root_token']) if not test_config: test_config = lifecycle_utils.get_charm_config() - del test_config['target_deploy_status']['vault'] + try: + del test_config['target_deploy_status']['vault'] + except KeyError: + # Already removed + pass zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) @@ -153,19 +160,31 @@ def test_csr(self): allowed_domains='openstack.local') test_config = lifecycle_utils.get_charm_config() - del test_config['target_deploy_status']['vault'] - zaza.model.block_until_file_has_contents( + try: + del test_config['target_deploy_status']['vault'] + except KeyError: + # Already removed + pass + zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', - zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, cacert.decode().strip()) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) ip = zaza.model.get_app_ips( 'keystone')[0] + with tempfile.NamedTemporaryFile(mode='w') as fp: fp.write(cacert.decode()) fp.flush() - requests.get('https://{}:5000'.format(ip), verify=fp.name) + # Avoid race condition and retry + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=2, min=2, max=10)): + with attempt: + logging.info( + "Attempting to connect to https://{}:5000".format(ip)) + requests.get('https://{}:5000'.format(ip), verify=fp.name) def test_all_clients_authenticated(self): """Check all vault clients are authenticated.""" @@ -245,8 +264,6 @@ def test_zzz_pause_resume(self): Pause service and check services are stopped, then resume and check they are started. """ - # Restarting vault process will set it as sealed so it's - # important to have the test executed at the end. vault_actions = zaza.model.get_actions( 'vault') if 'pause' not in vault_actions or 'resume' not in vault_actions: @@ -258,6 +275,80 @@ def test_zzz_pause_resume(self): lead_client = vault_utils.extract_lead_unit_client(self.clients) self.assertTrue(lead_client.hvac_client.seal_status['sealed']) + def test_vault_reload(self): + """Run reload tests. + + Reload service and check services were restarted + by doing simple change in the running config by API. + Then confirm that service is not sealed + """ + vault_actions = zaza.model.get_actions( + 'vault') + if 'reload' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have reload action") + + container_results = zaza.model.run_on_leader( + "vault", "systemd-detect-virt --container" + ) + container_rc = json.loads(container_results["Code"]) + if container_rc == 0: + raise unittest.SkipTest( + "Vault unit is running in a container. Cannot use mlock." + ) + + lead_client = vault_utils.get_cluster_leader(self.clients) + running_config = vault_utils.get_running_config(lead_client) + value_to_set = not running_config['data']['disable_mlock'] + + logging.info("Setting disable-mlock to {}".format(str(value_to_set))) + zaza.model.set_application_config( + 'vault', + {'disable-mlock': str(value_to_set)}) + + logging.info("Waiting for model to be idle ...") + zaza.model.block_until_all_units_idle(model_name=self.model_name) + + logging.info("Testing action reload on {}".format(lead_client)) + zaza.model.run_action( + juju_utils.get_unit_name_from_ip_address( + lead_client.addr, 'vault'), + 'reload', + model_name=self.model_name) + + logging.info("Getting new value ...") + new_value = vault_utils.get_running_config(lead_client)[ + 'data']['disable_mlock'] + + logging.info( + "Asserting new value {} is equal to set value {}" + .format(new_value, value_to_set)) + self.assertEqual( + value_to_set, + new_value) + + logging.info("Asserting not sealed") + self.assertFalse(lead_client.hvac_client.seal_status['sealed']) + + def test_vault_restart(self): + """Run pause and resume tests. + + Restart service and check services are started. + """ + vault_actions = zaza.model.get_actions( + 'vault') + if 'restart' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have restart action") + logging.info("Testing restart") + zaza.model.run_action_on_leader( + 'vault', + 'restart', + action_params={}) + + lead_client = vault_utils.extract_lead_unit_client(self.clients) + self.assertTrue(lead_client.hvac_client.seal_status['sealed']) + if __name__ == '__main__': unittest.main() diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 98142431c..40e33ac56 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -137,6 +137,41 @@ def get_vip_client(cacert=None): return client +def get_cluster_leader(clients): + """Get Vault cluster leader. + + We have to make sure we run api calls against the actual leader. + + :param clients: Clients list to get leader + :type clients: List of CharmVaultClient + :returns: CharmVaultClient + :rtype: CharmVaultClient or None + """ + if len(clients) == 1: + return clients[0] + + for client in clients: + if client.hvac_client.ha_status['is_self']: + return client + return None + + +def get_running_config(client): + """Get Vault running config. + + The hvac library does not support getting info from endpoint + /v1/sys/config/state/sanitized Therefore we implement it here + + :param client: Client used to get config + :type client: CharmVaultClient + :returns: dict from Vault api response + :rtype: dict + """ + return requests.get( + client.hvac_client.adapter.base_uri + '/v1/sys/config/state/sanitized', + headers={'X-Vault-Token': client.hvac_client.token}).json() + + def init_vault(client, shares=1, threshold=1): """Initialise vault. diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 35b5a143c..5798f2650 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -13,3 +13,155 @@ # limitations under the License. """Collection of utilities to support zaza tests etc.""" + + +import time + +from keystoneauth1.exceptions.connection import ConnectFailure + + +class ObjectRetrierWraps(object): + """An automatic retrier for an object. + + This is designed to be used with an instance of an object. Basically, it + wraps the object and any attributes that are fetched. Essentially, it is + used to provide retries on method calls on openstack client objects in + tests to increase robustness of tests. + + Although, technically this is bad, retries can be logged with the optional + log method. + + Usage: + + # get a client that does 3 retries, waits 5 seconds between retries and + # retries on any error. + some_client = ObjectRetrierWraps(get_some_client) + # this gets retried up to 3 times. + things = some_client.list_things() + + Note, it is quite simple. It wraps the object and on a getattr(obj, name) + it finds the name and then returns a wrapped version of that name. On a + call, it returns the value of that call. It only wraps objects in the + chain that are either callable or have a __getattr__() method. i.e. one + that can then be retried or further fetched. This means that if a.b.c() is + a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be + wrapped that the 'c' object __call__() method will be the one that is + actually retried. + + Note: this means that properties that do method calls won't be retried. + This is a limitation that may be addressed in the future, if it is needed. + """ + + def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0, + max_interval=15.0, total_wait=30.0, retry_exceptions=None, + log=None): + """Initialise the retrier object. + + :param obj: The object to wrap. Ought to be an instance of something + that you want to get methods on to call or be called itself. + :type obj: Any + :param num_retries: The (maximum) number of retries. May not be hit if + the total_wait time is exceeded. + :type num_retries: int + :param initial_interval: The initial or starting interval between + retries. + :type initial_interval: float + :param backoff: The exponential backoff multiple. 1 is linear. + :type backoff: float + :param max_interval: The maximum interval between retries. + If backoff is >1 then the initial_interval will never grow larger + than max_interval. + :type max_interval: float + :param retry_exceptions: The list of exceptions to retry on, or None. + If a list, then it will only retry if the exception is one of the + ones in the list. + :type retry_exceptions: List[Exception] + """ + # Note we use semi-private variable names that shouldn't clash with any + # on the actual object. + self.__obj = obj + self.__kwargs = { + 'num_retries': num_retries, + 'initial_interval': initial_interval, + 'backoff': backoff, + 'max_interval': max_interval, + 'total_wait': total_wait, + 'retry_exceptions': retry_exceptions, + 'log': log or (lambda x: None), + } + + def __getattr__(self, name): + """Get attribute; delegates to wrapped object.""" + # Note the above may generate an attribute error; we expect this and + # will fail with an attribute error. + attr = getattr(self.__obj, name) + if callable(attr) or hasattr(attr, "__getattr__"): + return ObjectRetrierWraps(attr, **self.__kwargs) + else: + return attr + # TODO(ajkavanagh): Note detecting a property is a bit trickier. we + # can do isinstance(attr, property), but then the act of accessing it + # is what calls it. i.e. it would fail at the getattr(self.__obj, + # name) stage. The solution is to check first, and if it's a property, + # then treat it like the retrier. However, I think this is too + # complex for the first go, and to use manual retries in that instance. + + def __call__(self, *args, **kwargs): + """Call the object; delegates to the wrapped object.""" + obj = self.__obj + retry = 0 + wait = self.__kwargs['initial_interval'] + max_interval = self.__kwargs['max_interval'] + log = self.__kwargs['log'] + backoff = self.__kwargs['backoff'] + total_wait = self.__kwargs['total_wait'] + num_retries = self.__kwargs['num_retries'] + retry_exceptions = self.__kwargs['retry_exceptions'] + wait_so_far = 0 + while True: + try: + return obj(*args, **kwargs) + except Exception as e: + # if retry_exceptions is not None, or the type of the exception + # is not in the list of retries, then raise an exception + # immediately. This means that if retry_exceptions is None, + # then the method is always retried. + if (retry_exceptions is not None and + type(e) not in retry_exceptions): + raise + retry += 1 + if retry > num_retries: + log("{}: exceeded number of retries, so erroring out" + .format(str(obj))) + raise e + log("{}: call failed: retrying in {} seconds" + .format(str(obj), wait)) + time.sleep(wait) + wait_so_far += wait + if wait_so_far >= total_wait: + raise e + wait = wait * backoff + if wait > max_interval: + wait = max_interval + + +def retry_on_connect_failure(client, **kwargs): + """Retry an object that eventually gets resolved to a call. + + Specifically, this uses ObjectRetrierWraps but only against the + keystoneauth1.exceptions.connection.ConnectFailure exeception. + + :params client: the object that may throw and exception when called. + :type client: Any + :params **kwargs: the arguments supplied to the ObjectRetrierWraps init + method + :type **kwargs: Dict[Any] + :returns: client wrapped in an ObjectRetrierWraps instance + :rtype: ObjectRetrierWraps[client] + """ + kwcopy = kwargs.copy() + if 'retry_exceptions' not in kwcopy: + kwcopy['retry_exceptions'] = [] + if ConnectFailure not in kwcopy['retry_exceptions']: + kwcopy['retry_exceptions'].append(ConnectFailure) + return ObjectRetrierWraps(client, **kwcopy) diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 6613154b3..a01f56e37 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -3,7 +3,7 @@ import logging import zaza.model as zaza_model -import zaza.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -225,7 +225,7 @@ def get_pools_from_broker_req(application_or_unit, model_name=None): """ # NOTE: we do not pass on a name for the remote_interface_name as that # varies between the Ceph consuming applications. - relation_data = zaza_juju.get_relation_from_unit( + relation_data = juju_utils.get_relation_from_unit( 'ceph-mon', application_or_unit, None, model_name=model_name) # NOTE: we probably should consume the Ceph broker code from c-h but c-h is diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index dd8b1e09a..fcd798960 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -20,6 +20,7 @@ import socket import subprocess import telnetlib +import tempfile import yaml from zaza import model @@ -680,3 +681,28 @@ def get_mojo_cacert_path(): return cacert else: raise zaza_exceptions.CACERTNotFound("Could not find cacert.pem") + + +def attach_file_resource(application_name, resource_name, + file_content, file_suffix=".txt"): + """Attaches a file as a Juju resource given the file content and suffix. + + The file content will be written into a temporary file with the given + suffix, and it will be attached to the Juju application. + + :param application_name: Juju application name. + :type application_name: string + :param resource_name: Juju resource name. + :type resource_name: string + :param file_content: The content of the file that will be attached + :type file_content: string + :param file_suffix: File suffix. This should be used to set the file + extension for applications that are sensitive to this. + :type file_suffix: string + :returns: None + """ + with tempfile.NamedTemporaryFile(mode='w', suffix=file_suffix) as fp: + fp.write(file_content) + fp.flush() + model.attach_resource( + application_name, resource_name, fp.name) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 35972c629..30820683e 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -16,9 +16,31 @@ This module contains a number of functions for interacting with OpenStack. """ +import collections +import copy +import datetime +import enum +import io +import itertools +import juju_wait +import logging +import os +import paramiko +import re +import shutil +import six +import subprocess +import sys +import tempfile +import tenacity +import textwrap +import urllib + + from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, + OVN_CODENAMES, PACKAGE_CODENAMES, OPENSTACK_RELEASES_PAIRS, ) @@ -41,27 +63,15 @@ import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env import zaza.utilities.juju as juju_utils +import zaza.utilities.maas from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient +from manilaclient import client as manilaclient -import datetime -import io -import itertools -import juju_wait -import logging -import os -import paramiko -import re -import six -import subprocess -import sys -import tempfile -import tenacity -import textwrap -import urllib +from juju.errors import JujuError import zaza @@ -123,6 +133,10 @@ 'pkg': 'ceph-common', 'origin_setting': 'source' }, + 'placement': { + 'pkg': 'placement-common', + 'origin_setting': 'openstack-origin' + }, } # Older tests use the order the services appear in the list to imply @@ -143,6 +157,7 @@ 'type': CHARM_TYPES['openstack-dashboard']}, {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']}, {'name': 'ceph-mon', 'type': CHARM_TYPES['ceph-mon']}, + {'name': 'placement', 'type': CHARM_TYPES['placement']}, ] @@ -169,20 +184,83 @@ 'ceilometer and gnocchi')}} # For vault TLS certificates +CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" +CERT_PROVIDERS = ['vault'] +REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) -KEYSTONE_LOCAL_CACERT = ("tests/{}".format(KEYSTONE_CACERT)) + + +async def async_block_until_ca_exists(application_name, ca_cert, + model_name=None, timeout=2700): + """Block until a CA cert is on all units of application_name. + + :param application_name: Name of application to check + :type application_name: str + :param ca_cert: The certificate content. + :type ca_cert: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: How long in seconds to wait + :type timeout: int + """ + async def _check_ca_present(model, ca_files): + units = model.applications[application_name].units + for ca_file in ca_files: + for unit in units: + try: + output = await unit.run('cat {}'.format(ca_file)) + contents = output.data.get('results').get('Stdout', '') + if ca_cert not in contents: + break + # libjuju throws a generic error for connection failure. So we + # cannot differentiate between a connectivity issue and a + # target file not existing error. For now just assume the + # latter. + except JujuError: + break + else: + # The CA was found in `ca_file` on all units. + return True + else: + return False + ca_files = await _async_get_remote_ca_cert_file_candidates( + application_name, + model_name=model_name) + async with zaza.model.run_in_model(model_name) as model: + await zaza.model.async_block_until( + lambda: _check_ca_present(model, ca_files), timeout=timeout) + +block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) + + +def get_cacert_absolute_path(filename): + """Build string containing location of the CA Certificate file. + + :param filename: Expected filename for CA Certificate file. + :type filename: str + :returns: Absolute path to file containing CA Certificate + :rtype: str + """ + return os.path.join( + deployment_env.get_tmpdir(), filename) def get_cacert(): """Return path to CA Certificate bundle for verification during test. :returns: Path to CA Certificate bundle or None. - :rtype: Optional[str] + :rtype: Union[str, None] """ - if os.path.exists(KEYSTONE_LOCAL_CACERT): - return KEYSTONE_LOCAL_CACERT + for _provider in CERT_PROVIDERS: + _cert = get_cacert_absolute_path( + CACERT_FILENAME_FORMAT.format(_provider)) + if os.path.exists(_cert): + return _cert + _keystone_local_cacert = get_cacert_absolute_path(KEYSTONE_CACERT) + if os.path.exists(_keystone_local_cacert): + return _keystone_local_cacert # OpenStack Client helpers @@ -248,15 +326,17 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session): +def get_nova_session_client(session, version=2): """Return novaclient authenticated by keystone session. :param session: Keystone session object :type session: keystoneauth1.session.Session object + :param version: Version of client to request. + :type version: float :returns: Authenticated novaclient :rtype: novaclient.Client object """ - return novaclient_client.Client(2, session=session) + return novaclient_client.Client(version, session=session) def get_neutron_session_client(session): @@ -370,6 +450,19 @@ def get_aodh_session_client(session): return aodh_client.Client(session=session) +def get_manila_session_client(session, version='2'): + """Return Manila client authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :param version: Manila API version + :type version: str + :returns: Authenticated manilaclient + :rtype: manilaclient.Client + """ + return manilaclient.Client(session=session, client_version=version) + + def get_keystone_scope(model_name=None): """Return Keystone scope based on OpenStack release of the overcloud. @@ -711,51 +804,74 @@ def add_interface_to_netplan(server_name, mac_address): model.run_on_unit(unit_name, "sudo netplan apply") -def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, - add_dataport_to_netplan=False, - limit_gws=None, - use_juju_wait=True): - """Configure the neturong-gateway external port. +class OpenStackNetworkingTopology(enum.Enum): + """OpenStack Charms Network Topologies.""" + + ML2_OVS = 'ML2+OVS' + ML2_OVS_DVR = 'ML2+OVS+DVR' + ML2_OVS_DVR_SNAT = 'ML2+OVS+DVR, no dedicated GWs' + ML2_OVN = 'ML2+OVN' + + +CharmedOpenStackNetworkingData = collections.namedtuple( + 'CharmedOpenStackNetworkingData', + [ + 'topology', + 'application_names', + 'unit_machine_ids', + 'port_config_key', + 'other_config', + ]) + + +def get_charm_networking_data(limit_gws=None): + """Inspect Juju model, determine networking topology and return data. - :param novaclient: Authenticated novaclient - :type novaclient: novaclient.Client object - :param neutronclient: Authenticated neutronclient - :type neutronclient: neutronclient.Client object - :param net_id: Network ID - :type net_id: string :param limit_gws: Limit the number of gateways that get a port attached :type limit_gws: Optional[int] - :param use_juju_wait: Whether to use juju wait to wait for the model to - settle once the gateway has been configured. Default is True - :type use_juju_wait: boolean - """ - deprecated_extnet_mode = deprecated_external_networking() - - port_config_key = 'data-port' - if deprecated_extnet_mode: - port_config_key = 'ext-port' + :rtype: CharmedOpenStackNetworkingData[ + OpenStackNetworkingTopology, + List[str], + Iterator[str], + str, + Dict[str,str]] + :returns: Named Tuple with networking data, example: + CharmedOpenStackNetworkingData( + OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis', 'ovn-dedicated-chassis'], + ['machine-id-1', 'machine-id-2'], # generator object + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'}) + :raises: RuntimeError + """ + # Initialize defaults, these will be amended to fit the reality of the + # model in the checks below. + topology = OpenStackNetworkingTopology.ML2_OVS + other_config = {} + port_config_key = ( + 'data-port' if not deprecated_external_networking() else 'ext-port') + unit_machine_ids = [] + application_names = [] - config = {} if dvr_enabled(): - uuids = itertools.islice(itertools.chain(get_ovs_uuids(), - get_gateway_uuids()), - limit_gws) - # If dvr, do not attempt to persist nic in netplan - # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 - add_dataport_to_netplan = False - application_names = ['neutron-openvswitch'] - try: - ngw = 'neutron-gateway' - model.get_application(ngw) - application_names.append(ngw) - except KeyError: - # neutron-gateway not in deployment - pass + if ngw_present(): + application_names = ['neutron-gateway', 'neutron-openvswitch'] + topology = OpenStackNetworkingTopology.ML2_OVS_DVR + else: + application_names = ['neutron-openvswitch'] + topology = OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT + unit_machine_ids = itertools.islice( + itertools.chain( + get_ovs_uuids(), + get_gateway_uuids()), + limit_gws) elif ngw_present(): - uuids = itertools.islice(get_gateway_uuids(), limit_gws) + unit_machine_ids = itertools.islice( + get_gateway_uuids(), limit_gws) application_names = ['neutron-gateway'] elif ovn_present(): - uuids = itertools.islice(get_ovn_uuids(), limit_gws) + topology = OpenStackNetworkingTopology.ML2_OVN + unit_machine_ids = itertools.islice(get_ovn_uuids(), limit_gws) application_names = ['ovn-chassis'] try: ovn_dc_name = 'ovn-dedicated-chassis' @@ -765,22 +881,50 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, # ovn-dedicated-chassis not in deployment pass port_config_key = 'bridge-interface-mappings' - config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) - add_dataport_to_netplan = True + other_config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) else: raise RuntimeError('Unable to determine charm network topology.') - if not net_id: - net_id = get_admin_net(neutronclient)['id'] + return CharmedOpenStackNetworkingData( + topology, + application_names, + unit_machine_ids, + port_config_key, + other_config) + + +def create_additional_port_for_machines(novaclient, neutronclient, net_id, + unit_machine_ids, + add_dataport_to_netplan=False): + """Create additional port for machines for use with external networking. - ports_created = 0 - for uuid in uuids: + :param novaclient: Undercloud Authenticated novaclient. + :type novaclient: novaclient.Client object + :param neutronclient: Undercloud Authenticated neutronclient. + :type neutronclient: neutronclient.Client object + :param net_id: Network ID to create ports on. + :type net_id: string + :param unit_machine_ids: Juju provider specific machine IDs for which we + should add ports on. + :type unit_machine_ids: Iterator[str] + :param add_dataport_to_netplan: Whether the newly created port should be + added to instance system configuration so + that it is brought up on instance reboot. + :type add_dataport_to_netplan: Optional[bool] + :returns: List of MAC addresses for created ports. + :rtype: List[str] + :raises: RuntimeError + """ + eligible_machines = 0 + for uuid in unit_machine_ids: + eligible_machines += 1 server = novaclient.servers.get(uuid) ext_port_name = "{}_ext-port".format(server.name) for port in neutronclient.list_ports(device_id=server.id)['ports']: if port['name'] == ext_port_name: logging.warning( - 'Neutron Gateway already has additional port') + 'Instance {} already has additional port, skipping.' + .format(server.id)) break else: logging.info('Attaching additional port to instance ("{}"), ' @@ -795,57 +939,134 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, } } port = neutronclient.create_port(body=body_value) - ports_created += 1 server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) if add_dataport_to_netplan: mac_address = get_mac_from_port(port, neutronclient) add_interface_to_netplan(server.name, mac_address=mac_address) - if not ports_created: - # NOTE: uuids is an iterator so testing it for contents or length prior - # to iterating over it is futile. + if not eligible_machines: + # NOTE: unit_machine_ids may be an iterator so testing it for contents + # or length prior to iterating over it is futile. raise RuntimeError('Unable to determine UUIDs for machines to attach ' 'external networking to.') - ext_br_macs = [] - for port in neutronclient.list_ports(network_id=net_id)['ports']: - if 'ext-port' in port['name']: - if deprecated_extnet_mode: - ext_br_macs.append(port['mac_address']) - else: - ext_br_macs.append('br-ex:{}'.format(port['mac_address'])) - ext_br_macs.sort() - ext_br_macs_str = ' '.join(ext_br_macs) - - if ext_br_macs: - config.update({port_config_key: ext_br_macs_str}) - for application_name in application_names: - logging.info('Setting {} on {}'.format( - config, application_name)) - current_data_port = get_application_config_option(application_name, - port_config_key) - if current_data_port == ext_br_macs_str: - logging.info('Config already set to value') - return - - model.set_application_config( - application_name, - configuration=config) - # NOTE(fnordahl): We are stuck with juju_wait until we figure out how - # to deal with all the non ['active', 'idle', 'Unit is ready.'] - # workload/agent states and msgs that our mojo specs are exposed to. - if use_juju_wait: - juju_wait.wait(wait_for_workload=True) - else: - zaza.model.wait_for_agent_status() - # TODO: shouldn't access get_charm_config() here as it relies on - # ./tests/tests.yaml existing by default (regardless of the - # fatal=False) ... it's not great design. - test_config = zaza.charm_lifecycle.utils.get_charm_config( - fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + # Retrieve the just created ports from Neutron so that we can provide our + # caller with their MAC addresses. + return [ + port['mac_address'] + for port in neutronclient.list_ports(network_id=net_id)['ports'] + if 'ext-port' in port['name'] + ] + + +def configure_networking_charms(networking_data, macs, use_juju_wait=True): + """Configure external networking for networking charms. + + :param networking_data: Data on networking charm topology. + :type networking_data: CharmedOpenStackNetworkingData + :param macs: MAC addresses of ports for use with external networking. + :type macs: Iterator[str] + :param use_juju_wait: Whether to use juju wait to wait for the model to + settle once the gateway has been configured. Default is True + :type use_juju_wait: Optional[bool] + """ + br_mac_fmt = 'br-ex:{}' if not deprecated_external_networking() else '{}' + br_mac = [ + br_mac_fmt.format(mac) + for mac in macs + ] + + config = copy.deepcopy(networking_data.other_config) + config.update({networking_data.port_config_key: ' '.join(sorted(br_mac))}) + + for application_name in networking_data.application_names: + logging.info('Setting {} on {}'.format( + config, application_name)) + current_data_port = get_application_config_option( + application_name, + networking_data.port_config_key) + if current_data_port == config[networking_data.port_config_key]: + logging.info('Config already set to value') + return + + model.set_application_config( + application_name, + configuration=config) + # NOTE(fnordahl): We are stuck with juju_wait until we figure out how + # to deal with all the non ['active', 'idle', 'Unit is ready.'] + # workload/agent states and msgs that our mojo specs are exposed to. + if use_juju_wait: + juju_wait.wait(wait_for_workload=True) + else: + zaza.model.wait_for_agent_status() + # TODO: shouldn't access get_charm_config() here as it relies on + # ./tests/tests.yaml existing by default (regardless of the + # fatal=False) ... it's not great design. + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) + + +def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, + add_dataport_to_netplan=False, + limit_gws=None, + use_juju_wait=True): + """Configure the neturong-gateway external port. + + :param novaclient: Authenticated novaclient + :type novaclient: novaclient.Client object + :param neutronclient: Authenticated neutronclient + :type neutronclient: neutronclient.Client object + :param net_id: Network ID + :type net_id: string + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] + :param use_juju_wait: Whether to use juju wait to wait for the model to + settle once the gateway has been configured. Default is True + :type use_juju_wait: boolean + """ + networking_data = get_charm_networking_data(limit_gws=limit_gws) + if networking_data.topology in ( + OpenStackNetworkingTopology.ML2_OVS_DVR, + OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT): + # If dvr, do not attempt to persist nic in netplan + # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 + add_dataport_to_netplan = False + + if not net_id: + net_id = get_admin_net(neutronclient)['id'] + + macs = create_additional_port_for_machines( + novaclient, neutronclient, net_id, networking_data.unit_machine_ids, + add_dataport_to_netplan) + + if macs: + configure_networking_charms( + networking_data, macs, use_juju_wait=use_juju_wait) + + +def configure_charmed_openstack_on_maas(network_config, limit_gws=None): + """Configure networking charms for charm-based OVS config on MAAS provider. + + :param network_config: Network configuration as provided in environment. + :type network_config: Dict[str] + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] + """ + networking_data = get_charm_networking_data(limit_gws=limit_gws) + macs = [ + mim.mac + for mim in zaza.utilities.maas.get_macs_from_cidr( + zaza.utilities.maas.get_maas_client_from_juju_cloud_data( + zaza.model.get_cloud_data()), + network_config['external_net_cidr'], + link_mode=zaza.utilities.maas.LinkMode.LINK_UP) + ] + if macs: + configure_networking_charms( + networking_data, macs, use_juju_wait=False) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), @@ -1474,8 +1695,23 @@ def get_swift_codename(version): :returns: Codename for swift :rtype: string """ - codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] - return codenames[0] + return _get_special_codename(version, SWIFT_CODENAMES) + + +def get_ovn_codename(version): + """Determine OpenStack codename that corresponds to OVN version. + + :param version: Version of OVN + :type version: string + :returns: Codename for OVN + :rtype: string + """ + return _get_special_codename(version, OVN_CODENAMES) + + +def _get_special_codename(version, codenames): + found = [k for k, v in six.iteritems(codenames) if version in v] + return found[0] def get_os_code_info(package, pkg_version): @@ -1488,7 +1724,6 @@ def get_os_code_info(package, pkg_version): :returns: Codename for package :rtype: string """ - # {'code_num': entry, 'code_name': OPENSTACK_CODENAMES[entry]} # Remove epoch if it exists if ':' in pkg_version: pkg_version = pkg_version.split(':')[1:][0] @@ -1512,13 +1747,55 @@ def get_os_code_info(package, pkg_version): # < Liberty co-ordinated project versions if 'swift' in package: return get_swift_codename(vers) + elif 'ovn' in package: + return get_ovn_codename(vers) else: return OPENSTACK_CODENAMES[vers] +def get_openstack_release(application, model_name=None): + """Return the openstack release codename based on /etc/openstack-release. + + This will only return a codename if the openstack-release package is + installed on the unit. + + :param application: Application name + :type application: string + :param model_name: Name of model to query. + :type model_name: str + :returns: OpenStack release codename for application + :rtype: string + """ + versions = [] + units = model.get_units(application, model_name=model_name) + for unit in units: + cmd = 'cat /etc/openstack-release | grep OPENSTACK_CODENAME' + try: + out = juju_utils.remote_run(unit.entity_id, cmd, + model_name=model_name) + except model.CommandRunFailed: + logging.debug('Fall back to version check for OpenStack codename') + else: + codename = out.split('=')[1].strip() + versions.append(codename) + if len(set(versions)) == 0: + return None + elif len(set(versions)) > 1: + raise Exception('Unexpected mix of OpenStack releases for {}: {}', + application, versions) + return versions[0] + + def get_current_os_versions(deployed_applications, model_name=None): """Determine OpenStack codename of deployed applications. + Initially, see if the openstack-release pkg is available and use it + instead. + + If it isn't then it falls back to the existing method of checking the + version of the package passed and then resolving the version from that + using lookup tables. + :param deployed_applications: List of deployed applications :type deployed_applications: list :param model_name: Name of model to query. @@ -1530,12 +1807,18 @@ def get_current_os_versions(deployed_applications, model_name=None): for application in UPGRADE_SERVICES: if application['name'] not in deployed_applications: continue + logging.info("looking at application: {}".format(application)) - version = generic_utils.get_pkg_version(application['name'], - application['type']['pkg'], - model_name=model_name) - versions[application['name']] = ( - get_os_code_info(application['type']['pkg'], version)) + codename = get_openstack_release(application['name'], + model_name=model_name) + if codename: + versions[application['name']] = codename + else: + version = generic_utils.get_pkg_version(application['name'], + application['type']['pkg'], + model_name=model_name) + versions[application['name']] = ( + get_os_code_info(application['type']['pkg'], version)) return versions @@ -1578,15 +1861,19 @@ def get_current_os_release_pair(application='keystone'): return '{}_{}'.format(series, os_version) -def get_os_release(release_pair=None): +def get_os_release(release_pair=None, application='keystone'): """Return index of release in OPENSTACK_RELEASES_PAIRS. + :param release_pair: OpenStack release pair eg 'focal_ussuri' + :type release_pair: string + :param application: Name of application to derive release pair from. + :type application: string :returns: Index of the release :rtype: int :raises: exceptions.ReleasePairNotFound """ if release_pair is None: - release_pair = get_current_os_release_pair() + release_pair = get_current_os_release_pair(application=application) try: index = OPENSTACK_RELEASES_PAIRS.index(release_pair) except ValueError: @@ -1787,29 +2074,82 @@ def get_overcloud_auth(address=None, model_name=None): 'OS_PROJECT_DOMAIN_NAME': 'admin_domain', 'API_VERSION': 3, } - if tls_rid: - unit = model.get_first_unit_name('keystone', model_name=model_name) - - # ensure that the path to put the local cacert in actually exists. The - # assumption that 'tests/' exists for, say, mojo is false. - # Needed due to: - # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 - _dir = os.path.dirname(KEYSTONE_LOCAL_CACERT) - if not os.path.exists(_dir): - os.makedirs(_dir) - - model.scp_from_unit( - unit, - KEYSTONE_REMOTE_CACERT, - KEYSTONE_LOCAL_CACERT) - - if os.path.exists(KEYSTONE_LOCAL_CACERT): - os.chmod(KEYSTONE_LOCAL_CACERT, 0o644) - auth_settings['OS_CACERT'] = KEYSTONE_LOCAL_CACERT + local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name) + if local_ca_cert: + auth_settings['OS_CACERT'] = local_ca_cert return auth_settings +async def _async_get_remote_ca_cert_file_candidates(application, + model_name=None): + """Return a list of possible remote CA file names. + + :param application: Name of application to examine. + :type application: str + :param model_name: Name of model to query. + :type model_name: str + :returns: List of paths to possible ca files. + :rtype: List[str] + """ + cert_files = [] + for _provider in CERT_PROVIDERS: + tls_rid = await model.async_get_relation_id( + application, + _provider, + model_name=model_name, + remote_interface_name='certificates') + if tls_rid: + cert_files.append( + REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( + _provider)) + cert_files.append(KEYSTONE_REMOTE_CACERT) + return cert_files + +_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper( + _async_get_remote_ca_cert_file_candidates) + + +def get_remote_ca_cert_file(application, model_name=None): + """Collect CA certificate from application. + + :param application: Name of application to collect file from. + :type application: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Path to cafile + :rtype: str + """ + unit = model.get_first_unit_name(application, model_name=model_name) + local_cert_file = None + cert_files = _get_remote_ca_cert_file_candidates( + application, + model_name=model_name) + for cert_file in cert_files: + _local_cert_file = get_cacert_absolute_path( + os.path.basename(cert_file)) + with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca: + try: + model.scp_from_unit( + unit, + cert_file, + _tmp_ca.name) + except JujuError: + continue + # ensure that the path to put the local cacert in actually exists. + # The assumption that 'tests/' exists for, say, mojo is false. + # Needed due to: + # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 + _dir = os.path.dirname(_local_cert_file) + if not os.path.exists(_dir): + os.makedirs(_dir) + shutil.move(_tmp_ca.name, _local_cert_file) + os.chmod(_local_cert_file, 0o644) + local_cert_file = _local_cert_file + break + return local_cert_file + + def get_urllib_opener(): """Create a urllib opener taking into account proxy settings. @@ -1881,7 +2221,8 @@ def download_image(image_url, target_file): def _resource_reaches_status(resource, resource_id, expected_status='available', - msg='resource'): + msg='resource', + resource_attribute='status'): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -1896,20 +2237,22 @@ def _resource_reaches_status(resource, resource_id, :param expected_status: status to expect resource to reach :type expected_status: str :param msg: text to identify purpose in logging - :type msy: str + :type msg: str + :param resource_attribute: Resource attribute to check against + :type resource_attribute: str :raises: AssertionError """ - resource_status = resource.get(resource_id).status - logging.info(resource_status) - assert resource_status == expected_status, ( - "Resource in {} state, waiting for {}" .format(resource_status, - expected_status,)) + resource_status = getattr(resource.get(resource_id), resource_attribute) + logging.info("{}: resource {} in {} state, waiting for {}".format( + msg, resource_id, resource_status, expected_status)) + assert resource_status == expected_status def resource_reaches_status(resource, resource_id, expected_status='available', msg='resource', + resource_attribute='status', wait_exponential_multiplier=1, wait_iteration_max_time=60, stop_after_attempt=8, @@ -1929,6 +2272,8 @@ def resource_reaches_status(resource, :type expected_status: str :param msg: text to identify purpose in logging :type msg: str + :param resource_attribute: Resource attribute to check against + :type resource_attribute: str :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier seconds between each retry :type wait_exponential_multiplier: int @@ -1950,7 +2295,8 @@ def resource_reaches_status(resource, resource, resource_id, expected_status, - msg) + msg, + resource_attribute) def _resource_removed(resource, resource_id, msg="resource"): @@ -1965,8 +2311,8 @@ def _resource_removed(resource, resource_id, msg="resource"): :raises: AssertionError """ matching = [r for r in resource.list() if r.id == resource_id] - logging.debug("Resource {} still present".format(resource_id)) - assert len(matching) == 0, "Resource {} still present".format(resource_id) + logging.debug("{}: resource {} still present".format(msg, resource_id)) + assert len(matching) == 0 def resource_removed(resource, @@ -2060,7 +2406,8 @@ def delete_volume_backup(cinder, vol_backup_id): def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', - visibility='public', container_format='bare'): + visibility='public', container_format='bare', + backend=None, force_import=False): """Upload the given image to glance and apply the given label. :param glance: Authenticated glanceclient @@ -2077,6 +2424,9 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', format that also contains metadata about the actual virtual machine. :type container_format: str + :param force_import: Force the use of glance image import + instead of direct upload + :type force_import: boolean :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2086,7 +2436,15 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', disk_format=disk_format, visibility=visibility, container_format=container_format) - glance.images.upload(image.id, open(local_path, 'rb')) + + if force_import: + logging.info('Forcing image import') + glance.images.stage(image.id, open(local_path, 'rb')) + glance.images.image_import( + image.id, method='glance-direct', backend=backend) + else: + glance.images.upload( + image.id, open(local_path, 'rb'), backend=backend) resource_reaches_status( glance.images, @@ -2098,7 +2456,9 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], - properties=None): + properties=None, backend=None, disk_format='qcow2', + visibility='public', container_format='bare', + force_import=False): """Download the image and upload it to glance. Download an image from image_url and upload it to glance labelling @@ -2117,6 +2477,9 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], :type tags: list of str :param properties: Properties and values to add to image :type properties: dict + :param force_import: Force the use of glance image import + instead of direct upload + :type force_import: boolean :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2130,9 +2493,16 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], local_path = os.path.join(image_cache_dir, img_name) if not os.path.exists(local_path): + logging.info('Downloading {} ...'.format(image_url)) download_image(image_url, local_path) + else: + logging.info('Cached image found at {} - Skipping download'.format( + local_path)) - image = upload_image_to_glance(glance, local_path, image_name) + image = upload_image_to_glance( + glance, local_path, image_name, backend=backend, + disk_format=disk_format, visibility=visibility, + container_format=container_format, force_import=force_import) for tag in tags: result = glance.image_tags.update(image.id, tag) logging.debug( @@ -2197,6 +2567,40 @@ def attach_volume(nova, volume_id, instance_id): device='/dev/vdx') +def failover_cinder_volume_host(cinder, backend_name='cinder-ceph', + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over'): + """Failover Cinder volume host with replication enabled. + + :param cinder: Authenticated cinderclient + :type cinder: cinder.Client + :param backend_name: Cinder volume backend name with + replication enabled. + :type backend_name: str + :param target_backend_id: Failover target Cinder backend id. + :type target_backend_id: str + :param target_status: Target Cinder volume status after failover. + :type target_status: str + :param target_replication_status: Target Cinder volume replication + status after failover. + :type target_replication_status: str + :raises: AssertionError + """ + host = 'cinder@{}'.format(backend_name) + logging.info('Failover Cinder volume host %s to backend_id %s', + host, target_backend_id) + cinder.services.failover_host(host=host, backend_id=target_backend_id) + for attempt in tenacity.Retrying( + retry=tenacity.retry_if_exception_type(AssertionError), + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + assert svc.status == target_status + assert svc.replication_status == target_replication_status + + def create_volume_backup(cinder, volume_id, name=None): """Create cinder volume backup. diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 9ad4d5013..c279e5894 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -95,8 +95,8 @@ async def async_action_unit_upgrade(units, model_name=None): action_unit_upgrade = sync_wrapper(async_action_unit_upgrade) -def action_upgrade_group(applications, model_name=None): - """Upgrade units using action managed upgrades. +def action_upgrade_apps(applications, model_name=None): + """Upgrade units in the applications using action managed upgrades. Upgrade all units of the given applications using action managed upgrades. This involves the following process: @@ -142,6 +142,45 @@ def action_upgrade_group(applications, model_name=None): done.extend(target) + # Ensure that mysql-innodb-cluster has at least one R/W group (it can get + # into a state where all are R/O whilst it is sorting itself out after an + # openstack_upgrade + if "mysql-innodb-cluster" in applications: + block_until_mysql_innodb_cluster_has_rw(model_name) + + # Now we need to wait for the model to go back to idle. + zaza.model.block_until_all_units_idle(model_name) + + +async def async_block_until_mysql_innodb_cluster_has_rw(model=None, + timeout=None): + """Block until the mysql-innodb-cluster is in a healthy state. + + Curiously, after a series of pauses and restarts (e.g. during an upgrade) + the mysql-innodb-cluster charms may not yet have agreed which one is the + R/W node; i.e. they are all R/O. Anyway, eventually they sort it out and + one jumps to the front and says "it's me!". This is detected, externally, + by the status line including R/W in the output. + + This function blocks until that happens so that no charm attempts to have a + chat with the mysql server before it has settled, thus breaking the whole + test. + """ + async def async_check_workload_messages_for_rw(model=None): + """Return True if a least one work message contains R/W.""" + status = await zaza.model.async_get_status() + app_status = status.applications.get("mysql-innodb-cluster") + units_data = app_status.units.values() + workload_statuses = [d.workload_status.info for d in units_data] + return any("R/W" in s for s in workload_statuses) + + await zaza.model.async_block_until(async_check_workload_messages_for_rw, + timeout=timeout) + + +block_until_mysql_innodb_cluster_has_rw = sync_wrapper( + async_block_until_mysql_innodb_cluster_has_rw) + def set_upgrade_application_config(applications, new_source, action_managed=True, model_name=None): @@ -150,7 +189,7 @@ def set_upgrade_application_config(applications, new_source, Set the charm config for upgrade. :param applications: List of application names. - :type applications: [] + :type applications: List[str] :param new_source: New package origin. :type new_source: str :param action_managed: Whether to set action-managed-upgrade config option. @@ -180,8 +219,8 @@ def set_upgrade_application_config(applications, new_source, def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. - :param new_source: New package origin. - :type new_source: str + :param app: The application to check + :type app: str :param model_name: Name of model to query. :type model_name: str :returns: Whether app be upgraded using action managed upgrade method. @@ -196,66 +235,95 @@ def is_action_upgradable(app, model_name=None): return supported -def run_action_upgrade(group, new_source, model_name=None): +def is_already_upgraded(app, new_src, model_name=None): + """Return True if the app has already been upgraded. + + :param app: The application to check + :type app: str + :param new_src: the new source (distro, cloud:x-y, etc.) + :type new_src: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Whether app be upgraded using action managed upgrade method. + :rtype: bool + """ + config = zaza.model.get_application_config(app, model_name=model_name) + try: + src = config['openstack-origin']['value'] + key_was = 'openstack-origin' + except KeyError: + src = config['source']['value'] + key_was = 'source' + logging.info("origin for {} is {}={}".format(app, key_was, src)) + return src == new_src + + +def run_action_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using action upgrades. - :param group: List of applications to upgrade. - :type group + :param apps: List of applications to upgrade. + :type apps: List[str] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - set_upgrade_application_config(group, new_source, model_name=model_name) - action_upgrade_group(group, model_name=model_name) + set_upgrade_application_config(apps, new_source, model_name=model_name) + action_upgrade_apps(apps, model_name=model_name) -def run_all_in_one_upgrade(group, new_source, model_name=None): +def run_all_in_one_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using all-in-one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: List[str] :source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ set_upgrade_application_config( - group, + apps, new_source, model_name=model_name, action_managed=False) zaza.model.block_until_all_units_idle() -def run_upgrade(group, new_source, model_name=None): +def run_upgrade_on_apps(apps, new_source, model_name=None): """Upgrade payload of all applications in group. Upgrade apps using action managed upgrades where possible and fallback to all_in_one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: [] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - action_upgrade = [] - all_in_one_upgrade = [] - for app in group: + action_upgrades = [] + all_in_one_upgrades = [] + for app in apps: + if is_already_upgraded(app, new_source, model_name=model_name): + logging.info("Application '%s' is already upgraded. Skipping.", + app) + continue if is_action_upgradable(app, model_name=model_name): - action_upgrade.append(app) + action_upgrades.append(app) else: - all_in_one_upgrade.append(app) - run_all_in_one_upgrade( - all_in_one_upgrade, - new_source, - model_name=model_name) - run_action_upgrade( - action_upgrade, - new_source, - model_name=model_name) + all_in_one_upgrades.append(app) + if all_in_one_upgrades: + run_all_in_one_upgrades( + all_in_one_upgrades, + new_source, + model_name=model_name) + if action_upgrades: + run_action_upgrades( + action_upgrades, + new_source, + model_name=model_name) def run_upgrade_tests(new_source, model_name=None): @@ -270,8 +338,6 @@ def run_upgrade_tests(new_source, model_name=None): :type model_name: str """ groups = get_upgrade_groups(model_name=model_name) - run_upgrade(groups['Core Identity'], new_source, model_name=model_name) - run_upgrade(groups['Storage'], new_source, model_name=model_name) - run_upgrade(groups['Control Plane'], new_source, model_name=model_name) - run_upgrade(groups['Compute'], new_source, model_name=model_name) - run_upgrade(groups['sweep_up'], new_source, model_name=model_name) + for name, apps in groups: + logging.info("Performing upgrade of %s", name) + run_upgrade_on_apps(apps, new_source, model_name=model_name) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 75dc72370..beaedec5e 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -36,6 +36,7 @@ ('eoan', 'train'), ('focal', 'ussuri'), ('groovy', 'victoria'), + ('hirsute', 'wallaby'), ]) @@ -69,9 +70,9 @@ 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', 'eoan_train', 'bionic_ussuri', 'focal_ussuri', - 'focal_victoria', 'groovy_victoria'] + 'focal_victoria', 'groovy_victoria', + 'focal_wallaby', 'hirsute_wallaby'] -# The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ ('diablo', ['1.4.3']), @@ -113,6 +114,17 @@ ['2.25.0']), ]) +OVN_CODENAMES = OrderedDict([ + ('train', + ['2.12']), + ('ussuri', + ['20.03']), + ('victoria', + ['20.06']), + ('wallaby', + ['20.12']), +]) + # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ @@ -245,10 +257,6 @@ ('10', 'ussuri'), ('11', 'victoria'), ]), - 'ovn-common': OrderedDict([ - ('2', 'train'), - ('20', 'ussuri'), - ]), 'ceph-common': OrderedDict([ ('10', 'mitaka'), # jewel ('12', 'queens'), # luminous @@ -256,4 +264,9 @@ ('14', 'train'), # nautilus ('15', 'ussuri'), # octopus ]), + 'placement-common': OrderedDict([ + ('2', 'train'), + ('3', 'ussuri'), + ('4', 'victoria'), + ]), } diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index aa6ab0e02..610496dea 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -58,7 +58,9 @@ def app_config(charm_name): } exceptions = { 'rabbitmq-server': { - 'origin': 'source', + # NOTE: AJK disable config-changed on rabbitmq-server due to bug: + # #1896520 + 'origin': None, 'pause_non_leader_subordinate': False, 'post_application_upgrade_functions': [ ('zaza.openstack.charm_tests.rabbitmq_server.utils.' @@ -94,7 +96,7 @@ def app_config(charm_name): 'pause_non_leader_subordinate': True, 'post_upgrade_functions': [ ('zaza.openstack.charm_tests.vault.setup.' - 'async_mojo_unseal_by_unit')] + 'async_mojo_or_default_unseal_by_unit')] }, 'mongodb': { 'origin': None, @@ -191,49 +193,64 @@ async def parallel_series_upgrade( status = (await model.async_get_status()).applications[application] logging.info( "Configuring leader / non leaders for {}".format(application)) - leader, non_leaders = get_leader_and_non_leaders(status) - for leader_name, leader_unit in leader.items(): + leaders, non_leaders = get_leader_and_non_leaders(status) + for leader_unit in leaders.values(): leader_machine = leader_unit["machine"] - leader = leader_name - machines = [ - unit["machine"] for name, unit - in non_leaders.items() - if unit['machine'] not in completed_machines] + machines = [unit["machine"] for name, unit in non_leaders.items() + if unit['machine'] not in completed_machines] await maybe_pause_things( status, non_leaders, pause_non_leader_subordinate, pause_non_leader_primary) - await series_upgrade_utils.async_set_series( - application, to_series=to_series) - app_idle = [ + # wait for the entire application set to be idle before starting upgrades + await asyncio.gather(*[ model.async_wait_for_unit_idle(unit, include_subordinates=True) - for unit in status["units"] - ] - await asyncio.gather(*app_idle) + for unit in status["units"]]) await prepare_series_upgrade(leader_machine, to_series=to_series) - prepare_group = [ - prepare_series_upgrade(machine, to_series=to_series) - for machine in machines] - await asyncio.gather(*prepare_group) + await asyncio.gather(*[ + wait_for_idle_then_prepare_series_upgrade( + machine, to_series=to_series) + for machine in machines]) if leader_machine not in completed_machines: machines.append(leader_machine) - upgrade_group = [ + await asyncio.gather(*[ series_upgrade_machine( machine, origin=origin, application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) - for machine in machines - ] - await asyncio.gather(*upgrade_group) + for machine in machines]) completed_machines.extend(machines) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) await run_post_application_upgrade_functions( post_application_upgrade_functions) +async def wait_for_idle_then_prepare_series_upgrade( + machine, to_series, model_name=None): + """Wait for the units to idle the do prepare_series_upgrade. + + We need to be sure that all the units are idle prior to actually calling + prepare_series_upgrade() as otherwise the call will fail. It has to be + checked because when the leader is paused it may kick off relation hooks in + the other units in an HA group. + + :param machine: the machine that is going to be prepared + :type machine: str + :param to_series: The series to which to upgrade + :type to_series: str + :param model_name: Name of model to query. + :type model_name: str + """ + await model.async_block_until_units_on_machine_are_idle( + machine, model_name=model_name) + await prepare_series_upgrade(machine, to_series=to_series) + + async def serial_series_upgrade( application, from_series='xenial', @@ -307,8 +324,10 @@ async def serial_series_upgrade( non_leaders, pause_non_leader_subordinate, pause_non_leader_primary) + logging.info("Finishing pausing application: {}".format(application)) await series_upgrade_utils.async_set_series( application, to_series=to_series) + logging.info("Finished set series for application: {}".format(application)) if not follower_first and leader_machine not in completed_machines: await model.async_wait_for_unit_idle(leader, include_subordinates=True) await prepare_series_upgrade(leader_machine, to_series=to_series) @@ -321,6 +340,8 @@ async def serial_series_upgrade( files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(leader_machine) + logging.info("Finished upgrading of leader for application: {}" + .format(application)) # for machine in machines: for unit_name, unit in non_leaders.items(): @@ -339,6 +360,8 @@ async def serial_series_upgrade( files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) + logging.info("Finished upgrading non leaders for application: {}" + .format(application)) if follower_first and leader_machine not in completed_machines: await model.async_wait_for_unit_idle(leader, include_subordinates=True) @@ -354,6 +377,7 @@ async def serial_series_upgrade( completed_machines.append(leader_machine) await run_post_application_upgrade_functions( post_application_upgrade_functions) + logging.info("Done series upgrade for: {}".format(application)) async def series_upgrade_machine( @@ -381,17 +405,16 @@ async def series_upgrade_machine( :returns: None :rtype: None """ - logging.info( - "About to series-upgrade ({})".format(machine)) + logging.info("About to series-upgrade ({})".format(machine)) await run_pre_upgrade_functions(machine, pre_upgrade_functions) await add_confdef_file(machine) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await remove_confdef_file(machine) await reboot(machine) + await series_upgrade_utils.async_complete_series_upgrade(machine) if origin: await os_utils.async_set_origin(application, origin) - await series_upgrade_utils.async_complete_series_upgrade(machine) await run_post_upgrade_functions(post_upgrade_functions) @@ -484,8 +507,7 @@ async def maybe_pause_things( :returns: Nothing :trype: None """ - subordinate_pauses = [] - leader_pauses = [] + unit_pauses = [] for unit in units: if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): @@ -495,15 +517,19 @@ async def maybe_pause_things( logging.info("Skipping pausing {} - blacklisted" .format(subordinate)) else: - logging.info("Pausing {}".format(subordinate)) - subordinate_pauses.append(model.async_run_action( - subordinate, "pause", action_params={})) + unit_pauses.append( + _pause_helper("subordinate", subordinate)) if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - leader_pauses.append( - model.async_run_action(unit, "pause", action_params={})) - await asyncio.gather(*leader_pauses) - await asyncio.gather(*subordinate_pauses) + unit_pauses.append(_pause_helper("leader", unit)) + if unit_pauses: + await asyncio.gather(*unit_pauses) + + +async def _pause_helper(_type, unit): + """Pause helper to ensure that the log happens nearer to the action.""" + logging.info("Pausing ({}) {}".format(_type, unit)) + await model.async_run_action(unit, "pause", action_params={}) + logging.info("Finished Pausing ({}) {}".format(_type, unit)) def get_leader_and_non_leaders(status): @@ -541,14 +567,14 @@ async def prepare_series_upgrade(machine, to_series): NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series - :param machine_num: Machine number - :type machine_num: str + :param machine: Machine number + :type machine: str :param to_series: The series to which to upgrade :type to_series: str :returns: None :rtype: None """ - logging.info("Preparing series upgrade for: {}".format(machine)) + logging.info("Preparing series upgrade for: %s", machine) await series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) @@ -564,9 +590,8 @@ async def reboot(machine): try: await model.async_run_on_machine(machine, 'sudo init 6 & exit') # await run_on_machine(unit, "sudo reboot && exit") - except subprocess.CalledProcessError as e: - logging.warn("Error doing reboot: {}".format(e)) - pass + except subprocess.CalledProcessError as error: + logging.warning("Error doing reboot: %s", error) async def async_dist_upgrade(machine): @@ -577,16 +602,31 @@ async def async_dist_upgrade(machine): :returns: None :rtype: None """ - logging.info('Updating package db ' + machine) + logging.info('Updating package db %s', machine) update_cmd = 'sudo apt-get update' await model.async_run_on_machine(machine, update_cmd) - logging.info('Updating existing packages ' + machine) + logging.info('Updating existing packages %s', machine) dist_upgrade_cmd = ( """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_machine(machine, dist_upgrade_cmd) + rdict = await model.async_run_on_machine( + machine, + "cat /var/run/reboot-required || true") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot machine: %s", machine) + await reboot(machine) + logging.info("Waiting for machine to come back afer reboot: %s", + machine) + await model.async_block_until_file_missing_on_machine( + machine, "/var/run/reboot-required") + logging.info("Waiting for machine idleness on %s", machine) + await asyncio.sleep(5.0) + await model.async_block_until_units_on_machine_are_idle(machine) + # TODO: change this to wait on units on the machine + # await model.async_block_until_all_units_idle() async def async_do_release_upgrade(machine): @@ -597,7 +637,7 @@ async def async_do_release_upgrade(machine): :returns: None :rtype: None """ - logging.info('Upgrading ' + machine) + logging.info('Upgrading %s', machine) do_release_upgrade_cmd = ( 'yes | sudo DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 97ba1539e..42683f6f5 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -884,7 +884,8 @@ def dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd) - rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required") + rdict = model.run_on_unit(unit_name, + "cat /var/run/reboot-required || true") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) os_utils.reboot(unit_name) @@ -919,8 +920,8 @@ async def async_dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_unit(unit_name, dist_upgrade_cmd) - rdict = await model.async_run_on_unit(unit_name, - "cat /var/run/reboot-required") + rdict = await model.async_run_on_unit( + unit_name, "cat /var/run/reboot-required || true") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) await os_utils.async_reboot(unit_name) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 995e0bbc9..4f005926e 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -19,8 +19,20 @@ import re import zaza.model - - +import zaza.utilities.juju +from zaza.openstack.utilities.os_versions import ( + OPENSTACK_CODENAMES, + UBUNTU_OPENSTACK_RELEASE, + OPENSTACK_RELEASES_PAIRS, +) + +""" +The below upgrade order is surfaced in end-user documentation. Any change to +it should be accompanied by an update to the OpenStack Charms Deployment Guide +for both charm upgrades and payload upgrades: +- source/upgrade-charms.rst#upgrade-order +- source/upgrade-openstack.rst#openstack_upgrade_order +""" SERVICE_GROUPS = ( ('Database Services', ['percona-cluster', 'mysql-innodb-cluster']), ('Stateful Services', ['rabbitmq-server', 'ceph-mon']), @@ -46,7 +58,7 @@ def get_upgrade_candidates(model_name=None, filters=None): :param filters: List of filter functions to apply :type filters: List[fn] :returns: List of application that can have their payload upgraded. - :rtype: [] + :rtype: Dict[str, Dict[str, ANY]] """ if filters is None: filters = [] @@ -163,8 +175,8 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None): :param model_name: Name of model to query. :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: collections.OrderedDict + :returns: List of tuples(group name, applications) + :rtype: List[Tuple[str, Dict[str, ANY]]] """ filters = [_filter_subordinates] filters = _apply_extra_filters(filters, extra_filters) @@ -226,3 +238,110 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] + + +def get_all_principal_applications(model_name=None): + """Return a list of all the prinical applications in the model. + + :param model_name: Optional model name + :type model_name: Optional[str] + :returns: List of principal application names + :rtype: List[str] + """ + status = zaza.utilities.juju.get_full_juju_status(model_name=model_name) + return [application for application in status.applications.keys() + if not status.applications.get(application)['subordinate-to']] + + +def get_lowest_openstack_version(current_versions): + """Get the lowest OpenStack version from the list of current versions. + + :param current_versions: The list of versions + :type current_versions: List[str] + :returns: the lowest version currently installed. + :rtype: str + """ + lowest_version = 'zebra' + for svc in current_versions.keys(): + if current_versions[svc] < lowest_version: + lowest_version = current_versions[svc] + return lowest_version + + +def determine_next_openstack_release(release): + """Determine the next release after the one passed as a str. + + The returned value is a tuple of the form: ('2020.1', 'ussuri') + + :param release: the release to use as the base + :type release: str + :returns: the release tuple immediately after the current one. + :rtype: Tuple[str, str] + :raises: KeyError if the current release doesn't actually exist + """ + old_index = list(OPENSTACK_CODENAMES.values()).index(release) + new_index = old_index + 1 + return list(OPENSTACK_CODENAMES.items())[new_index] + + +def determine_new_source(ubuntu_version, current_source, new_release, + single_increment=True): + """Determine the new source/openstack-origin value based on new release. + + This takes the ubuntu_version and the current_source (in the form of + 'distro' or 'cloud:xenial-mitaka') and either converts it to a new source, + or returns None if the new_release will match the current_source (i.e. it's + already at the right release), or it's simply not possible. + + If single_increment is set, then the returned source will only be returned + if the new_release is one more than the release in the current source. + + :param ubuntu_version: the ubuntu version that the app is installed on. + :type ubuntu_version: str + :param current_source: a source in the form of 'distro' or + 'cloud:xenial-mitaka' + :type current_source: str + :param new_release: a new OpenStack version codename. e.g. 'stein' + :type new_release: str + :param single_increment: If True, only allow single increment upgrade. + :type single_increment: boolean + :returns: The new source in the form of 'cloud:bionic-train' or None if not + possible + :rtype: Optional[str] + :raises: KeyError if any of the strings don't correspond to known values. + """ + logging.warn("determine_new_source: locals: %s", locals()) + if current_source == 'distro': + # convert to a ubuntu-openstack pair + current_source = "cloud:{}-{}".format( + ubuntu_version, UBUNTU_OPENSTACK_RELEASE[ubuntu_version]) + # strip out the current openstack version + if ':' not in current_source: + current_source = "cloud:{}-{}".format(ubuntu_version, current_source) + pair = current_source.split(':')[1] + u_version, os_version = pair.split('-', 2) + if u_version != ubuntu_version: + logging.warn("determine_new_source: ubuntu_versions don't match: " + "%s != %s" % (ubuntu_version, u_version)) + return None + # determine versions + openstack_codenames = list(OPENSTACK_CODENAMES.values()) + old_index = openstack_codenames.index(os_version) + try: + new_os_version = openstack_codenames[old_index + 1] + except IndexError: + logging.warn("determine_new_source: no OpenStack version after " + "'%s'" % os_version) + return None + if single_increment and new_release != new_os_version: + logging.warn("determine_new_source: requested version '%s' not a " + "single increment from '%s' which is '%s'" % ( + new_release, os_version, new_os_version)) + return None + # now check that there is a combination of u_version-new_os_version + new_pair = "{}_{}".format(u_version, new_os_version) + if new_pair not in OPENSTACK_RELEASES_PAIRS: + logging.warn("determine_new_source: now release pair candidate for " + " combination cloud:%s-%s" % (u_version, new_os_version)) + return None + return "cloud:{}-{}".format(u_version, new_os_version)