diff --git a/salt-shaptools.changes b/salt-shaptools.changes index 71cbbbfe..b7a85add 100644 --- a/salt-shaptools.changes +++ b/salt-shaptools.changes @@ -1,3 +1,9 @@ +------------------------------------------------------------------- +Fri Mar 20 14:49:04 UTC 2020 - Xabier Arbulu + +- Version 0.3.3 + * Add new salt state to extract the HANA python dbapi client + ------------------------------------------------------------------- Thu Mar 5 10:03:39 UTC 2020 - Xabier Arbulu diff --git a/salt-shaptools.spec b/salt-shaptools.spec index f9d04bbc..6f6f6bc6 100644 --- a/salt-shaptools.spec +++ b/salt-shaptools.spec @@ -19,7 +19,7 @@ # See also https://en.opensuse.org/openSUSE:Specfile_guidelines Name: salt-shaptools -Version: 0.3.2 +Version: 0.3.3 Release: 0 Summary: Salt modules and states for SAP Applications and SLE-HA components management diff --git a/salt/modules/hanamod.py b/salt/modules/hanamod.py index 2f120e58..44d87c0d 100644 --- a/salt/modules/hanamod.py +++ b/salt/modules/hanamod.py @@ -22,9 +22,19 @@ # Import Python libs from __future__ import absolute_import, unicode_literals, print_function + +import logging import time +import re + +try: # pragma: no cover + import importlib as imp +except ImportError: # pragma: no cover + import imp from salt import exceptions +from salt.utils import files as salt_files + # Import third party libs try: @@ -35,8 +45,19 @@ except ImportError: # pragma: no cover HAS_HANA = False +LOGGER = logging.getLogger(__name__) + __virtualname__ = 'hana' +LABEL_FILE = 'LABEL.ASC' +LABELIDX_FILE = 'LABELIDX.ASC' + + +class SapFolderNotFoundError(Exception): + ''' + SAP folder not found exception + ''' + def __virtual__(): # pragma: no cover ''' @@ -865,17 +886,17 @@ def wait_for_connection( ''' Wait until HANA is ready trying to connect to the database - host: + host Host where HANA is running - port: + port HANA database port - user: + user User to connect to the databse - password: + password Password to connect to the database - timeout: + timeout Timeout to try to connect to the database - interval: + interval Interval to try the connection CLI Example: @@ -900,4 +921,75 @@ def wait_for_connection( raise exceptions.CommandExecutionError( 'HANA database not available after {} seconds in {}:{}'.format( timeout, host, port - )) \ No newline at end of file + )) + + +def reload_hdb_connector(): + ''' + As hdb_connector uses pyhdb or dbapi, if these packages are installed on the fly, + we need to reload the connector to import the correct api + ''' + imp.reload(hdb_connector) + + +def _find_sap_folder(software_folders, folder_pattern): + ''' + Find a SAP folder following a recursive approach using the LABEL and LABELIDX files + ''' + for folder in software_folders: + label = '{}/{}'.format(folder, LABEL_FILE) + try: + with salt_files.fopen(label, 'r') as label_file_ptr: + label_content = label_file_ptr.read().strip() + if folder_pattern.match(label_content): + return folder + else: + LOGGER.debug( + '%s folder does not contain %s pattern', folder, folder_pattern.pattern) + except IOError: + LOGGER.debug('%s file not found in %s. Skipping folder', LABEL_FILE, folder) + + labelidx = '{}/{}'.format(folder, LABELIDX_FILE) + try: + with salt_files.fopen(labelidx, 'r') as labelidx_file_ptr: + labelidx_content = labelidx_file_ptr.read().splitlines() + new_folders = [ + '{}/{}'.format(folder, new_folder) for new_folder in labelidx_content] + try: + return _find_sap_folder(new_folders, folder_pattern) + except SapFolderNotFoundError: + continue + except IOError: + LOGGER.debug('%s file not found in %s. Skipping folder', LABELIDX_FILE, folder) + + raise SapFolderNotFoundError( + 'SAP folder with {} pattern not found'.format(folder_pattern.pattern)) + + +def extract_pydbapi( + name, + software_folders, + output_dir, + hana_version='20'): + ''' + Extract HANA pydbapi python client from the provided software folders + + name + Name of the package that needs to be installed + software_folders + Folders list where the HANA client is located. It's used as a list as the pydbapi client + will be found automatically among different folders and providing several folders is a + standard way in SAP landscape + output_dir + Folder where the package is extracted + ''' + current_platform = hana.HanaInstance.get_platform() + hana_client_pattern = re.compile('^HDB_CLIENT:{}.*:{}:.*'.format( + hana_version, current_platform)) + try: + hana_client_folder = _find_sap_folder(software_folders, hana_client_pattern) + except SapFolderNotFoundError: + raise exceptions.CommandExecutionError('HANA client not found') + pydbapi_file = '{}/client/{}'.format(hana_client_folder, name) + __salt__['archive.tar'](options='xvf', tarfile=pydbapi_file, dest=output_dir) + return pydbapi_file diff --git a/salt/states/hanamod.py b/salt/states/hanamod.py index dbebd7a8..469f0ad6 100644 --- a/salt/states/hanamod.py +++ b/salt/states/hanamod.py @@ -223,7 +223,7 @@ def installed( software_path=software_path, conf_file=TMP_CONFIG_FILE, root_user=root_user, - root_password=root_password) + root_password=root_password) if hdb_pwd_file: __salt__['cp.get_file']( path=hdb_pwd_file, @@ -755,3 +755,57 @@ def memory_resources_updated( except exceptions.CommandExecutionError as err: ret['comment'] = six.text_type(err) return ret + + +def pydbapi_extracted( + name, + software_folders, + output_dir, + hana_version='20', + force=False): + ''' + Extract HANA pydbapi python client from the provided software folders + + name + Name of the package that needs to be installed + software_folders + Folders list where the HANA client is located. It's used as a list as the pydbapi client + will be found automatically among different folders and providing several folders is a + standard way in SAP landscape + output_dir + Folder where the package is extracted + force + Force new extraction if the file already is extracted + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + if not force and __salt__['file.directory_exists'](output_dir): + ret['result'] = True + ret['comment'] = \ + '{} already exists. Skipping extraction (set force to True to force the '\ + 'extraction)'.format(output_dir) + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = '{} would be extracted'.format(name) + ret['changes']['output_dir'] = output_dir + return ret + + __salt__['file.mkdir'](output_dir) + + try: + client = __salt__['hana.extract_pydbapi'](name, software_folders, output_dir, hana_version) + except exceptions.CommandExecutionError as err: + ret['comment'] = six.text_type(err) + return ret + + ret['result'] = True + ret['comment'] = '{} correctly extracted'.format(client) + ret['changes'] = {'pydbapi': client, 'output_dir': output_dir} + + return ret diff --git a/tests/unit/modules/test_hanamod.py b/tests/unit/modules/test_hanamod.py index e990de8b..cc5f7dcb 100644 --- a/tests/unit/modules/test_hanamod.py +++ b/tests/unit/modules/test_hanamod.py @@ -17,6 +17,7 @@ from tests.support.mock import ( MagicMock, patch, + mock_open, NO_MOCK, NO_MOCK_REASON ) @@ -813,3 +814,106 @@ def test_wait_for_connection_error(self, mock_time, mock_sleep, mock_hdb_connect mock.call('192.168.10.15', 30015, user='SYSTEM', password='pass') ]) assert 'HANA database not available after 2 seconds in 192.168.10.15:30015' in str(err.value) + + @mock.patch('salt.modules.hanamod.hdb_connector') + @mock.patch('importlib.reload') + def test_reload_hdb_connector(self, mock_reload, mock_hdb_connector): + hanamod.reload_hdb_connector() + mock_reload.assert_called_once_with(mock_hdb_connector) + + @mock.patch('logging.Logger.debug') + @mock.patch('salt.utils.files.fopen') + def test_find_sap_folder_error(self, mock_fopen, mock_debug): + mock_pattern = mock.Mock(pattern='my_pattern') + mock_fopen.side_effect = [ + IOError, IOError, IOError, IOError] + with pytest.raises(hanamod.SapFolderNotFoundError) as err: + hanamod._find_sap_folder(['1234', '5678'], mock_pattern) + + assert 'SAP folder with my_pattern pattern not found' in str(err.value) + mock_debug.assert_has_calls([ + mock.call('%s file not found in %s. Skipping folder', 'LABEL.ASC', '1234'), + mock.call('%s file not found in %s. Skipping folder', 'LABELIDX.ASC', '1234'), + mock.call('%s file not found in %s. Skipping folder', 'LABEL.ASC', '5678'), + mock.call('%s file not found in %s. Skipping folder', 'LABELIDX.ASC', '5678') + ]) + + def test_find_sap_folder_contain_hana(self): + mock_pattern = mock.Mock(return_value=True) + with patch('salt.utils.files.fopen', mock_open(read_data='data\n')) as mock_file: + folder = hanamod._find_sap_folder(['1234', '5678'], mock_pattern) + + mock_pattern.match.assert_called_once_with('data') + assert folder in '1234' + + @mock.patch('logging.Logger.debug') + def test_find_sap_folder_contain_units(self, mock_debug): + mock_pattern = mock.Mock(pattern='my_pattern') + mock_pattern.match.side_effect = [False, True] + with patch('salt.utils.files.fopen', mock_open(read_data= + ['data\n', 'DATA_UNITS\n', 'data_2\n'])) as mock_file: + folder = hanamod._find_sap_folder(['1234', '5678'], mock_pattern) + + mock_pattern.match.assert_has_calls([ + mock.call('data'), + mock.call('data_2') + ]) + mock_debug.assert_has_calls([ + mock.call('%s folder does not contain %s pattern', '1234', 'my_pattern') + ]) + assert folder in '1234/DATA_UNITS' + + @mock.patch('logging.Logger.debug') + def test_find_sap_folder_contain_units_error(self, mock_debug): + mock_pattern = mock.Mock(pattern='my_pattern') + mock_pattern.match.side_effect = [False, False] + with patch('salt.utils.files.fopen', mock_open(read_data=[ + 'data\n', 'DATA_UNITS\n', 'data_2\n', IOError])) as mock_file: + with pytest.raises(hanamod.SapFolderNotFoundError) as err: + folder = hanamod._find_sap_folder(['1234'], mock_pattern) + + mock_pattern.match.assert_has_calls([ + mock.call('data'), + mock.call('data_2') + ]) + mock_debug.assert_has_calls([ + mock.call('%s folder does not contain %s pattern', '1234', 'my_pattern') + ]) + assert 'SAP folder with my_pattern pattern not found' in str(err.value) + + @mock.patch('re.compile') + @mock.patch('salt.modules.hanamod._find_sap_folder') + @mock.patch('salt.modules.hanamod.hana.HanaInstance.get_platform') + def test_extract_pydbapi(self, mock_get_platform, mock_find_sap_folders, mock_compile): + mock_get_platform.return_value = 'LINUX_X86_64' + mock_find_sap_folders.return_value = 'my_folder' + compile_mocked = mock.Mock() + mock_compile.return_value = compile_mocked + mock_tar = MagicMock() + with patch.dict(hanamod.__salt__, {'archive.tar': mock_tar}): + pydbapi_file = hanamod.extract_pydbapi( + 'PYDBAPI.tar.gz', ['1234', '5678'], '/tmp/output') + + mock_compile.assert_called_once_with('^HDB_CLIENT:20.*:LINUX_X86_64:.*') + mock_find_sap_folders.assert_called_once_with( + ['1234', '5678'], compile_mocked) + mock_tar.assert_called_once_with( + options='xvf', tarfile='my_folder/client/PYDBAPI.tar.gz', dest='/tmp/output') + assert pydbapi_file == 'my_folder/client/PYDBAPI.tar.gz' + + @mock.patch('re.compile') + @mock.patch('salt.modules.hanamod._find_sap_folder') + @mock.patch('salt.modules.hanamod.hana.HanaInstance.get_platform') + def test_extract_pydbapi_error(self, mock_get_platform, mock_find_sap_folders, mock_compile): + mock_get_platform.return_value = 'LINUX_X86_64' + compile_mocked = mock.Mock() + mock_compile.return_value = compile_mocked + mock_find_sap_folders.side_effect = hanamod.SapFolderNotFoundError + with pytest.raises(exceptions.CommandExecutionError) as err: + pydbapi_file = hanamod.extract_pydbapi( + 'PYDBAPI.tar.gz', ['1234', '5678'], '/tmp/output') + + mock_compile.assert_called_once_with('^HDB_CLIENT:20.*:LINUX_X86_64:.*') + mock_find_sap_folders.assert_called_once_with( + ['1234', '5678'], compile_mocked) + assert 'HANA client not found' in str(err.value) diff --git a/tests/unit/states/test_hanamod.py b/tests/unit/states/test_hanamod.py index 7d851666..97e49a8a 100644 --- a/tests/unit/states/test_hanamod.py +++ b/tests/unit/states/test_hanamod.py @@ -262,7 +262,7 @@ def test_installed_invalid_params(self): assert hanamod.installed( 'prd', '00', 'pass', '/software', 'root', 'pass') == ret - + mock_create.assert_called_once_with( software_path='/software', conf_file=hanamod.TMP_CONFIG_FILE, @@ -1041,3 +1041,64 @@ def test_memory_resources_updated_error(self): sid='prd', inst='00', password='pass') + + def test_pydbapi_extracted_already_exists(self): + ret = {'name': 'PYDBAPI.tar', + 'changes': {}, + 'result': True, + 'comment': '/tmp/output already exists. Skipping extraction (set force to True to force the extraction)'} + + mock_dir_exists = MagicMock(return_value=True) + + with patch.dict(hanamod.__salt__, {'file.directory_exists': mock_dir_exists}): + assert hanamod.pydbapi_extracted( + 'PYDBAPI.tar', ['1234', '5678'], '/tmp/output') == ret + + mock_dir_exists.assert_called_once_with('/tmp/output') + + def test_pydbapi_extracted_test(self): + ret = {'name': 'PYDBAPI.tar', + 'changes': {'output_dir': '/tmp/output'}, + 'result': None, + 'comment': 'PYDBAPI.tar would be extracted'} + + with patch.dict(hanamod.__opts__, {'test': True}): + assert hanamod.pydbapi_extracted( + 'PYDBAPI.tar', ['1234', '5678'], '/tmp/output', force=True) == ret + + def test_pydbapi_extracted_error(self): + ret = {'name': 'PYDBAPI.tar', + 'changes': {}, + 'result': False, + 'comment': 'error extracting'} + + mock_mkdir = MagicMock() + mock_extract_pydbapi = MagicMock( + side_effect=exceptions.CommandExecutionError('error extracting')) + + with patch.dict(hanamod.__salt__, {'file.mkdir': mock_mkdir, + 'hana.extract_pydbapi': mock_extract_pydbapi}): + assert hanamod.pydbapi_extracted( + 'PYDBAPI.tar', ['1234', '5678'], '/tmp/output', force=True) == ret + + mock_mkdir.assert_called_once_with('/tmp/output') + mock_extract_pydbapi.assert_called_once_with( + 'PYDBAPI.tar', ['1234', '5678'], '/tmp/output', '20') + + def test_pydbapi_extracted_correct(self): + ret = {'name': 'PYDBAPI.tar', + 'changes': {'pydbapi': 'py_client', 'output_dir': '/tmp/output'}, + 'result': True, + 'comment': 'py_client correctly extracted'} + + mock_mkdir = MagicMock() + mock_extract_pydbapi = MagicMock(return_value='py_client') + + with patch.dict(hanamod.__salt__, {'file.mkdir': mock_mkdir, + 'hana.extract_pydbapi': mock_extract_pydbapi}): + assert hanamod.pydbapi_extracted( + 'PYDBAPI.tar', ['1234', '5678'], '/tmp/output', force=True) == ret + + mock_mkdir.assert_called_once_with('/tmp/output') + mock_extract_pydbapi.assert_called_once_with( + 'PYDBAPI.tar', ['1234', '5678'], '/tmp/output', '20')