From 2db04eb07cfcfd818e5f067ddfe5363694ae7280 Mon Sep 17 00:00:00 2001 From: Xabier Arbulu Insausti Date: Wed, 4 Mar 2020 16:38:38 +0000 Subject: [PATCH 1/6] Create new code to configure corosync conf file --- salt/states/crmshmod.py | 108 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/salt/states/crmshmod.py b/salt/states/crmshmod.py index 0c5cc1b0..d9fd6c1e 100644 --- a/salt/states/crmshmod.py +++ b/salt/states/crmshmod.py @@ -35,6 +35,7 @@ # Import salt libs from salt import exceptions +from salt import utils as salt_utils from salt.ext import six @@ -287,3 +288,110 @@ def cluster_configured( except exceptions.CommandExecutionError as err: ret['comment'] = six.text_type(err) return ret + + +def _convert2dict(file_content_lines): + """ + Convert the corosync configuration file to a dictionary + """ + corodict = {} + index = 0 + + for i, line in enumerate(file_content_lines): + stripped_line = line.strip() + if not stripped_line or stripped_line[0] == '#': + continue + + if index > i: + continue + + line_items = stripped_line.split() + if '{' in stripped_line: + corodict[line_items[0]], new_index = _convert2dict(file_content_lines[i+1:]) + index = i + new_index + elif line_items[0][-1] == ':': + corodict[line_items[0][:-1]] = line_items[-1] + elif '}' in stripped_line: + return corodict, i+2 + + return corodict, index + + +def _mergedicts(main_dict, changes_dict, applied_changes, initial_path=''): + """ + Merge the 2 dictionaries. We cannot use update as it changes all the children of an entry + """ + for key, value in changes_dict.items(): + current_path = '{}.{}'.format(initial_path, key) + if key in main_dict.keys() and not isinstance(value, dict): + if str(main_dict[key]) != str(value): + applied_changes[current_path] = value + main_dict[key] = value + elif key in main_dict.keys(): + modified_dict, new_changes = _mergedicts(main_dict[key], value, applied_changes, current_path) + main_dict[key] = modified_dict + applied_changes.update(new_changes) + + else: # Entry not found in current main dictionary, so we can update all + main_dict[key] = changes_dict[key] + applied_changes[current_path] = value + + return main_dict, applied_changes + + +def _convert2corosync(corodict, indentation=''): + """ + Convert a corosync data dictionary to the corosync configuration file format + """ + output = '' + for key, value in corodict.items(): + if isinstance(value, dict): + output += '{}{} {{\n'.format(indentation, key) + indentation += '\t' + output += _convert2corosync(value, indentation) + indentation = indentation[:-1] + output += '{}}}\n'.format(indentation) + else: + output += '{}{}: {}\n'.format(indentation, key, value) + return output + + +def corosync_updated( + name, + data, + backup=True): + """ + Configure corosync configuration file + + name + Corosync configuration file path + data + Dictionary with the values that have to be changed. The method won't do any sanity check + so, it will put in the configuration file value added in this parameter + """ + + changes = {} + ret = {'name': name, + 'changes': changes, + 'result': False, + 'comment': ''} + + with salt_utils.files.fopen(name, 'r') as file_content: + corodict, _ = _convert2dict(file_content.read().splitlines()) + new_conf_dict, changes = _mergedicts(corodict, data, {}) + + if not changes: + ret['changes'] = changes + ret['comment'] = 'Corosync already has the required configuration' + ret['result'] = True + return ret + + new_conf_file_content = _convert2corosync(new_conf_dict) + if backup: + __salt__['file.copy'](name, '{}.backup'.format(name)) + __salt__['file.write'](name, new_conf_file_content) + + ret['changes'] = changes + ret['comment'] = 'Corosync configuration file updated' + ret['result'] = True + return ret From b8ce17e60afb55f35e796bba163b34a4bacb2062 Mon Sep 17 00:00:00 2001 From: Xabier Arbulu Insausti Date: Wed, 4 Mar 2020 16:40:50 +0000 Subject: [PATCH 2/6] Install salt in develop mode to reload the test files --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc5e49b9..b635ab08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ jobs: - git clone --depth=50 https://github.com/openSUSE/salt ../salt - rm ../salt/tests/conftest.py - git clone --depth=50 https://github.com/SUSE/shaptools.git ../shaptools - - pip install ../salt + - pip install -e ../salt - pip install ../shaptools script: - ./tests/run.sh @@ -34,7 +34,7 @@ jobs: - git clone --depth=50 https://github.com/openSUSE/salt ../salt - rm ../salt/tests/conftest.py - git clone --depth=50 https://github.com/SUSE/shaptools.git ../shaptools - - pip install ../salt + - pip install -e ../salt - pip install ../shaptools before_script: - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter From 8708e05431c34deb8d8f1b6a21d3a0069b180102 Mon Sep 17 00:00:00 2001 From: Xabier Arbulu Insausti Date: Thu, 5 Mar 2020 10:02:56 +0000 Subject: [PATCH 3/6] Add unit tests to the new corosync methods --- tests/unit/states/test_crmshmod.py | 236 +++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/tests/unit/states/test_crmshmod.py b/tests/unit/states/test_crmshmod.py index e76ff751..d6ca20c0 100644 --- a/tests/unit/states/test_crmshmod.py +++ b/tests/unit/states/test_crmshmod.py @@ -15,6 +15,7 @@ from tests.support.mock import ( NO_MOCK, NO_MOCK_REASON, + mock_open, MagicMock, patch ) @@ -480,3 +481,238 @@ def test_configured_command_error(self): method='update', url='file.config', is_xml=False) + + def test_convert2dict(self): + corofile = """ +# Please read the corosync.conf.5 manual page +totem { + version: 2 + max_messages: 20 + interface { + ringnumber: 0 + + } + transport: udpu +} + +logging { + timestamp: on + logger_subsys { + debug: off + } + +} + +quorum { + expected_votes: 1 + two_node: 0 +}""" + + corodict, _ = crmshmod._convert2dict(corofile.splitlines()) + + assert corodict == { + 'totem': { + 'version': '2', + 'max_messages': '20', + 'interface': { + 'ringnumber': '0' + }, + 'transport': 'udpu' + }, + 'logging': { + 'timestamp': 'on', + 'logger_subsys': { + 'debug': 'off' + } + }, + 'quorum': { + 'expected_votes': '1', + 'two_node': '0' + } + } + + def test_merge_dicts1(self): + main_dict = { + 'a': { + 'b': 1, + 'c': 2 + }, + 'd': 3 + } + changed_dict = { + 'a': { + 'c': 4 + }, + 'd': 5 + } + merged_dict, applied_changes = crmshmod._mergedicts( + main_dict, changed_dict, {}, '') + + assert merged_dict == { + 'a': { + 'b': 1, + 'c': 4 + }, + 'd': 5 + } + + assert applied_changes == { + '.a.c': 4, + '.d': 5 + } + + def test_merge_dicts2(self): + main_dict = { + 'a': { + 'b': { + 'f': 7 + }, + 'c': 2 + }, + 'd': 3 + } + changed_dict = { + 'a': { + 'b': { + 'f': 8 + }, + }, + 'd': 5 + } + merged_dict, applied_changes = crmshmod._mergedicts( + main_dict, changed_dict, {}, '') + + assert merged_dict == { + 'a': { + 'b': { + 'f': 8 + }, + 'c': 2 + }, + 'd': 5 + } + + assert applied_changes == { + '.d': 5, + '.a.b.f': 8 + } + + def test_merge_dicts3(self): + main_dict = { + 'a': { + 'b': 1, + 'c': 2 + }, + 'd': 3 + } + changed_dict = { + 'e': { + 'c': 4 + }, + 'a': { + 'b': 3 + }, + 'd': 5 + } + merged_dict, applied_changes = crmshmod._mergedicts( + main_dict, changed_dict, {}, '') + + assert merged_dict == { + 'a': { + 'b': 3, + 'c': 2 + }, + 'e': { + 'c': 4 + }, + 'd': 5 + } + + assert applied_changes == { + '.d': 5, + '.a.b': 3, + '.e': {'c': 4} + } + + def test_convert2corosync(self): + main_dict = { + 'a': { + 'b': { + 'f': 7 + }, + 'c': 2 + }, + 'd': 3 + } + + output = crmshmod._convert2corosync(main_dict, '') + assert output == "a {\n\tb {\n\t\tf: 7\n\t}\n\tc: 2\n}\nd: 3\n" + + @mock.patch('salt.states.crmshmod._convert2dict') + @mock.patch('salt.states.crmshmod._mergedicts') + def test_corosync_updated_already(self, mock_mergedicts, mock_convert2dict): + ''' + Test to check corosync_updated when configuration is already applied + ''' + + ret = {'name': '/etc/corosync/corosync.conf', + 'changes': {}, + 'result': True, + 'comment': 'Corosync already has the required configuration'} + + mock_convert2dict.return_value = ({'data': 1}, {}) + mock_mergedicts.return_value = ({}, {}) + + file_content = "my corosync file content\nmy corosync file 2nd line content" + with patch("salt.utils.files.fopen", mock_open(read_data=file_content)): + assert crmshmod.corosync_updated( + name='/etc/corosync/corosync.conf', + data={'my_data': 1}) == ret + + mock_convert2dict.assert_called_once_with( + ['my corosync file content', 'my corosync file 2nd line content'] + ) + mock_mergedicts.assert_called_once_with( + {'data': 1}, {'my_data': 1}, {}) + + @mock.patch('salt.states.crmshmod._convert2corosync') + @mock.patch('salt.states.crmshmod._convert2dict') + @mock.patch('salt.states.crmshmod._mergedicts') + def testt_corosync_updated(self, mock_mergedicts, mock_convert2dict, mock_convert2corosync): + ''' + Test to check corosync_updated when configuration is applied + ''' + + ret = {'name': '/etc/corosync/corosync.conf', + 'changes': {'change1': 1, 'change2': 2}, + 'result': True, + 'comment': 'Corosync configuration file updated'} + + mock_copy = MagicMock() + mock_write = MagicMock() + mock_convert2dict.return_value = ({'data': 1}, {}) + mock_mergedicts.return_value = ({'updated': 2}, {'change1': 1, 'change2': 2}) + mock_convert2corosync.return_value = 'new content' + + file_content = "my corosync file content\nmy corosync file 2nd line content" + + with patch.dict(crmshmod.__salt__, {'file.copy': mock_copy, + 'file.write': mock_write}): + with patch("salt.utils.files.fopen", mock_open(read_data=file_content)): + assert crmshmod.corosync_updated( + name='/etc/corosync/corosync.conf', + data={'my_data': 1}) == ret + + mock_convert2dict.assert_called_once_with( + ['my corosync file content', 'my corosync file 2nd line content'] + ) + mock_mergedicts.assert_called_once_with( + {'data': 1}, {'my_data': 1}, {}) + + mock_convert2corosync.assert_called_once_with({'updated': 2}) + + mock_copy.assert_called_once_with( + '/etc/corosync/corosync.conf', '/etc/corosync/corosync.conf.backup') + + mock_write.assert_called_once_with( + '/etc/corosync/corosync.conf', 'new content') From 36ba33e7185ce72578e864452e11a3ee6f507460 Mon Sep 17 00:00:00 2001 From: Xabier Arbulu Insausti Date: Thu, 5 Mar 2020 10:05:10 +0000 Subject: [PATCH 4/6] Upgrade version of the package and changes file --- salt-shaptools.changes | 8 ++++++++ salt-shaptools.spec | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt-shaptools.changes b/salt-shaptools.changes index 52c9a232..35f3b0a3 100644 --- a/salt-shaptools.changes +++ b/salt-shaptools.changes @@ -1,3 +1,11 @@ +------------------------------------------------------------------- +Thu Mar 5 10:03:39 UTC 2020 - Xabier Arbulu + +- Version 0.3.2 + * Add a new salt state method to update corosync configuration + file + * Fix travis file to install the py packages in develop mode + ------------------------------------------------------------------- Fri Jan 24 10:40:26 UTC 2020 - Xabier Arbulu diff --git a/salt-shaptools.spec b/salt-shaptools.spec index 458143b1..f9d04bbc 100644 --- a/salt-shaptools.spec +++ b/salt-shaptools.spec @@ -19,7 +19,7 @@ # See also https://en.opensuse.org/openSUSE:Specfile_guidelines Name: salt-shaptools -Version: 0.3.1 +Version: 0.3.2 Release: 0 Summary: Salt modules and states for SAP Applications and SLE-HA components management From 4990400bac3e9a842c2a0159c5a826ae9f7f538e Mon Sep 17 00:00:00 2001 From: Xabier Arbulu Insausti Date: Thu, 5 Mar 2020 10:07:03 +0000 Subject: [PATCH 5/6] Correct the last unit test method name --- tests/unit/states/test_crmshmod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/test_crmshmod.py b/tests/unit/states/test_crmshmod.py index d6ca20c0..b5a5d687 100644 --- a/tests/unit/states/test_crmshmod.py +++ b/tests/unit/states/test_crmshmod.py @@ -678,7 +678,7 @@ def test_corosync_updated_already(self, mock_mergedicts, mock_convert2dict): @mock.patch('salt.states.crmshmod._convert2corosync') @mock.patch('salt.states.crmshmod._convert2dict') @mock.patch('salt.states.crmshmod._mergedicts') - def testt_corosync_updated(self, mock_mergedicts, mock_convert2dict, mock_convert2corosync): + def test_corosync_updated(self, mock_mergedicts, mock_convert2dict, mock_convert2corosync): ''' Test to check corosync_updated when configuration is applied ''' From eef7159c8bf6f6ad74c6bfca8fb058ac43e1c223 Mon Sep 17 00:00:00 2001 From: Xabier Arbulu Insausti Date: Thu, 5 Mar 2020 10:27:28 +0000 Subject: [PATCH 6/6] Fix one UT to make it py2 compatible --- tests/unit/states/test_crmshmod.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/states/test_crmshmod.py b/tests/unit/states/test_crmshmod.py index b5a5d687..126924f5 100644 --- a/tests/unit/states/test_crmshmod.py +++ b/tests/unit/states/test_crmshmod.py @@ -6,6 +6,7 @@ # Import Python libs from __future__ import absolute_import, unicode_literals, print_function +import sys from salt import exceptions # Import Salt Testing Libs @@ -646,7 +647,13 @@ def test_convert2corosync(self): } output = crmshmod._convert2corosync(main_dict, '') - assert output == "a {\n\tb {\n\t\tf: 7\n\t}\n\tc: 2\n}\nd: 3\n" + + # Py2 and py3 have different way of ordering the `items` method + # For the functionality this does not really affect + if sys.version_info[0] == 2: + assert output == "a {\n\tc: 2\n\tb {\n\t\tf: 7\n\t}\n}\nd: 3\n" + else: + assert output == "a {\n\tb {\n\t\tf: 7\n\t}\n\tc: 2\n}\nd: 3\n" @mock.patch('salt.states.crmshmod._convert2dict') @mock.patch('salt.states.crmshmod._mergedicts')