Merge pull request #58 from arbulu89/feature/extract-pydbapi

Feature/extract pydbapi

salt-shaptools.changes

@@ -1,3 +1,9 @@
+-------------------------------------------------------------------
+Fri Mar 20 14:49:04 UTC 2020 - Xabier Arbulu <[email protected]>
+
+- Version 0.3.3
+  * Add new salt state to extract the HANA python dbapi client 
+
 -------------------------------------------------------------------
 Thu Mar  5 10:03:39 UTC 2020 - Xabier Arbulu <[email protected]>
 

salt-shaptools.spec

@@ -19,7 +19,7 @@
 # See also https://en.opensuse.org/openSUSE:Specfile_guidelines
 
 Name:           salt-shaptools
-Version:        0.3.2
+Version:        0.3.3
 Release:        0
 Summary:        Salt modules and states for SAP Applications and SLE-HA components management
 

salt/modules/hanamod.py

@@ -22,9 +22,19 @@
 
 # Import Python libs
 from __future__ import absolute_import, unicode_literals, print_function
+
+import logging
 import time
+import re
+
+try:  # pragma: no cover
+    import importlib as imp
+except ImportError:  # pragma: no cover
+    import imp
 
 from salt import exceptions
+from salt.utils import files as salt_files
+
 
 # Import third party libs
 try:
@@ -35,8 +45,19 @@
 except ImportError:  # pragma: no cover
     HAS_HANA = False
 
+LOGGER = logging.getLogger(__name__)
+
 __virtualname__ = 'hana'
 
+LABEL_FILE = 'LABEL.ASC'
+LABELIDX_FILE = 'LABELIDX.ASC'
+
+
+class SapFolderNotFoundError(Exception):
+    '''
+    SAP folder not found exception
+    '''
+
 
 def __virtual__():  # pragma: no cover
     '''
@@ -865,17 +886,17 @@ def wait_for_connection(
     '''
     Wait until HANA is ready trying to connect to the database
 
-    host:
+    host
         Host where HANA is running
-    port:
+    port
         HANA database port
-    user:
+    user
         User to connect to the databse
-    password:
+    password
         Password to connect to the database
-    timeout:
+    timeout
         Timeout to try to connect to the database
-    interval:
+    interval
         Interval to try the connection
 
     CLI Example:
@@ -900,4 +921,75 @@ def wait_for_connection(
         raise exceptions.CommandExecutionError(
             'HANA database not available after {} seconds in {}:{}'.format(
                 timeout, host, port
-            ))
+            ))
+
+
+def reload_hdb_connector():
+    '''
+    As hdb_connector uses pyhdb or dbapi, if these packages are installed on the fly,
+    we need to reload the connector to import the correct api
+    '''
+    imp.reload(hdb_connector)
+
+
+def _find_sap_folder(software_folders, folder_pattern):
+    '''
+    Find a SAP folder following a recursive approach using the LABEL and LABELIDX files
+    '''
+    for folder in software_folders:
+        label = '{}/{}'.format(folder, LABEL_FILE)
+        try:
+            with salt_files.fopen(label, 'r') as label_file_ptr:
+                label_content = label_file_ptr.read().strip()
+                if folder_pattern.match(label_content):
+                    return folder
+                else:
+                    LOGGER.debug(
+                        '%s folder does not contain %s pattern', folder, folder_pattern.pattern)
+        except IOError:
+            LOGGER.debug('%s file not found in %s. Skipping folder', LABEL_FILE, folder)
+
+        labelidx = '{}/{}'.format(folder, LABELIDX_FILE)
+        try:
+            with salt_files.fopen(labelidx, 'r') as labelidx_file_ptr:
+                labelidx_content = labelidx_file_ptr.read().splitlines()
+                new_folders = [
+                    '{}/{}'.format(folder, new_folder) for new_folder in labelidx_content]
+                try:
+                    return _find_sap_folder(new_folders, folder_pattern)
+                except SapFolderNotFoundError:
+                    continue
+        except IOError:
+            LOGGER.debug('%s file not found in %s. Skipping folder', LABELIDX_FILE, folder)
+
+    raise SapFolderNotFoundError(
+        'SAP folder with {} pattern not found'.format(folder_pattern.pattern))
+
+
+def extract_pydbapi(
+        name,
+        software_folders,
+        output_dir,
+        hana_version='20'):
+    '''
+    Extract HANA pydbapi python client from the provided software folders
+
+    name
+        Name of the package that needs to be installed
+    software_folders
+        Folders list where the HANA client is located. It's used as a list as the pydbapi client
+        will be found automatically among different folders and providing several folders is a
+        standard way in SAP landscape
+    output_dir
+        Folder where the package is extracted
+    '''
+    current_platform = hana.HanaInstance.get_platform()
+    hana_client_pattern = re.compile('^HDB_CLIENT:{}.*:{}:.*'.format(
+        hana_version, current_platform))
+    try:
+        hana_client_folder = _find_sap_folder(software_folders, hana_client_pattern)
+    except SapFolderNotFoundError:
+        raise exceptions.CommandExecutionError('HANA client not found')
+    pydbapi_file = '{}/client/{}'.format(hana_client_folder, name)
+    __salt__['archive.tar'](options='xvf', tarfile=pydbapi_file, dest=output_dir)
+    return pydbapi_file

salt/states/hanamod.py

@@ -223,7 +223,7 @@ def installed(
             software_path=software_path,
             conf_file=TMP_CONFIG_FILE,
             root_user=root_user,
-            root_password=root_password)        
+            root_password=root_password)
         if hdb_pwd_file:
             __salt__['cp.get_file'](
                 path=hdb_pwd_file,
@@ -755,3 +755,57 @@ def memory_resources_updated(
     except exceptions.CommandExecutionError as err:
         ret['comment'] = six.text_type(err)
         return ret
+
+
+def pydbapi_extracted(
+        name,
+        software_folders,
+        output_dir,
+        hana_version='20',
+        force=False):
+    '''
+    Extract HANA pydbapi python client from the provided software folders
+
+    name
+        Name of the package that needs to be installed
+    software_folders
+        Folders list where the HANA client is located. It's used as a list as the pydbapi client
+        will be found automatically among different folders and providing several folders is a
+        standard way in SAP landscape
+    output_dir
+        Folder where the package is extracted
+    force
+        Force new extraction if the file already is extracted
+    '''
+
+    ret = {'name': name,
+           'changes': {},
+           'result': False,
+           'comment': ''}
+
+    if not force and __salt__['file.directory_exists'](output_dir):
+        ret['result'] = True
+        ret['comment'] = \
+            '{} already exists. Skipping extraction (set force to True to force the '\
+            'extraction)'.format(output_dir)
+        return ret
+
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = '{} would be extracted'.format(name)
+        ret['changes']['output_dir'] = output_dir
+        return ret
+
+    __salt__['file.mkdir'](output_dir)
+
+    try:
+        client = __salt__['hana.extract_pydbapi'](name, software_folders, output_dir, hana_version)
+    except exceptions.CommandExecutionError as err:
+        ret['comment'] = six.text_type(err)
+        return ret
+
+    ret['result'] = True
+    ret['comment'] = '{} correctly extracted'.format(client)
+    ret['changes'] = {'pydbapi': client, 'output_dir': output_dir}
+
+    return ret

tests/unit/modules/test_hanamod.py

@@ -17,6 +17,7 @@
 from tests.support.mock import (
     MagicMock,
     patch,
+    mock_open,
     NO_MOCK,
     NO_MOCK_REASON
 )
@@ -813,3 +814,106 @@ def test_wait_for_connection_error(self, mock_time, mock_sleep, mock_hdb_connect
             mock.call('192.168.10.15', 30015, user='SYSTEM', password='pass')
         ])
         assert 'HANA database not available after 2 seconds in 192.168.10.15:30015' in str(err.value)
+
+    @mock.patch('salt.modules.hanamod.hdb_connector')
+    @mock.patch('importlib.reload')
+    def test_reload_hdb_connector(self, mock_reload, mock_hdb_connector):
+        hanamod.reload_hdb_connector()
+        mock_reload.assert_called_once_with(mock_hdb_connector)
+
+    @mock.patch('logging.Logger.debug')
+    @mock.patch('salt.utils.files.fopen')
+    def test_find_sap_folder_error(self, mock_fopen, mock_debug):
+        mock_pattern = mock.Mock(pattern='my_pattern')
+        mock_fopen.side_effect = [
+            IOError, IOError, IOError, IOError]
+        with pytest.raises(hanamod.SapFolderNotFoundError) as err:
+            hanamod._find_sap_folder(['1234', '5678'], mock_pattern)
+
+        assert 'SAP folder with my_pattern pattern not found' in str(err.value)
+        mock_debug.assert_has_calls([
+            mock.call('%s file not found in %s. Skipping folder', 'LABEL.ASC', '1234'),
+            mock.call('%s file not found in %s. Skipping folder', 'LABELIDX.ASC', '1234'),
+            mock.call('%s file not found in %s. Skipping folder', 'LABEL.ASC', '5678'),
+            mock.call('%s file not found in %s. Skipping folder', 'LABELIDX.ASC', '5678')
+        ])
+
+    def test_find_sap_folder_contain_hana(self):
+        mock_pattern = mock.Mock(return_value=True)
+        with patch('salt.utils.files.fopen', mock_open(read_data='data\n')) as mock_file:
+            folder = hanamod._find_sap_folder(['1234', '5678'], mock_pattern)
+
+        mock_pattern.match.assert_called_once_with('data')
+        assert folder in '1234'
+
+    @mock.patch('logging.Logger.debug')
+    def test_find_sap_folder_contain_units(self, mock_debug):
+        mock_pattern = mock.Mock(pattern='my_pattern')
+        mock_pattern.match.side_effect = [False, True]
+        with patch('salt.utils.files.fopen', mock_open(read_data=
+                ['data\n', 'DATA_UNITS\n', 'data_2\n'])) as mock_file:
+            folder = hanamod._find_sap_folder(['1234', '5678'], mock_pattern)
+
+        mock_pattern.match.assert_has_calls([
+            mock.call('data'),
+            mock.call('data_2')
+        ])
+        mock_debug.assert_has_calls([
+            mock.call('%s folder does not contain %s pattern', '1234', 'my_pattern')
+        ])
+        assert folder in '1234/DATA_UNITS'
+
+    @mock.patch('logging.Logger.debug')
+    def test_find_sap_folder_contain_units_error(self, mock_debug):
+        mock_pattern = mock.Mock(pattern='my_pattern')
+        mock_pattern.match.side_effect = [False, False]
+        with patch('salt.utils.files.fopen', mock_open(read_data=[
+                'data\n', 'DATA_UNITS\n', 'data_2\n', IOError])) as mock_file:
+            with pytest.raises(hanamod.SapFolderNotFoundError) as err:
+                folder = hanamod._find_sap_folder(['1234'], mock_pattern)
+
+        mock_pattern.match.assert_has_calls([
+            mock.call('data'),
+            mock.call('data_2')
+        ])
+        mock_debug.assert_has_calls([
+            mock.call('%s folder does not contain %s pattern', '1234', 'my_pattern')
+        ])
+        assert 'SAP folder with my_pattern pattern not found' in str(err.value)
+
+    @mock.patch('re.compile')
+    @mock.patch('salt.modules.hanamod._find_sap_folder')
+    @mock.patch('salt.modules.hanamod.hana.HanaInstance.get_platform')
+    def test_extract_pydbapi(self, mock_get_platform, mock_find_sap_folders, mock_compile):
+        mock_get_platform.return_value = 'LINUX_X86_64'
+        mock_find_sap_folders.return_value = 'my_folder'
+        compile_mocked = mock.Mock()
+        mock_compile.return_value = compile_mocked
+        mock_tar = MagicMock()
+        with patch.dict(hanamod.__salt__, {'archive.tar': mock_tar}):
+            pydbapi_file = hanamod.extract_pydbapi(
+                'PYDBAPI.tar.gz', ['1234', '5678'], '/tmp/output')
+
+        mock_compile.assert_called_once_with('^HDB_CLIENT:20.*:LINUX_X86_64:.*')
+        mock_find_sap_folders.assert_called_once_with(
+            ['1234', '5678'], compile_mocked)
+        mock_tar.assert_called_once_with(
+            options='xvf', tarfile='my_folder/client/PYDBAPI.tar.gz', dest='/tmp/output')
+        assert pydbapi_file == 'my_folder/client/PYDBAPI.tar.gz'
+
+    @mock.patch('re.compile')
+    @mock.patch('salt.modules.hanamod._find_sap_folder')
+    @mock.patch('salt.modules.hanamod.hana.HanaInstance.get_platform')
+    def test_extract_pydbapi_error(self, mock_get_platform, mock_find_sap_folders, mock_compile):
+        mock_get_platform.return_value = 'LINUX_X86_64'
+        compile_mocked = mock.Mock()
+        mock_compile.return_value = compile_mocked
+        mock_find_sap_folders.side_effect = hanamod.SapFolderNotFoundError
+        with pytest.raises(exceptions.CommandExecutionError) as err:
+            pydbapi_file = hanamod.extract_pydbapi(
+                'PYDBAPI.tar.gz', ['1234', '5678'], '/tmp/output')
+
+        mock_compile.assert_called_once_with('^HDB_CLIENT:20.*:LINUX_X86_64:.*')
+        mock_find_sap_folders.assert_called_once_with(
+            ['1234', '5678'], compile_mocked)
+        assert 'HANA client not found' in str(err.value)