Added the IBM resources to tagging

cloud_governance/common/clouds/ibm/account/ibm_account.py

@@ -33,7 +33,8 @@ def __init__(self):
         self.__API_KEY = self.__environment_variables_dict.get('IBM_API_KEY', '')
         try:
             if self.__API_KEY and self.__API_USERNAME:
-                self.__sl_client = SoftLayer.create_client_from_env(username=self.__API_USERNAME, api_key=self.__API_KEY, timeout=self.DELAY)
+                self.__sl_client = SoftLayer.create_client_from_env(username=self.__API_USERNAME,
+                                                                    api_key=self.__API_KEY, timeout=self.DELAY)
                 self.short_account_id = str(self.__API_USERNAME.split('_')[0])
             if self.__environment_variables_dict.get('USAGE_REPORTS_APIKEY'):
                 self.__service_client = UsageReportsV4.new_instance()
@@ -81,23 +82,32 @@ def get_user_tags_from_gsheet(self, username: str, user_email: str = '', file_pa
         if not os.path.exists(file_name):
             self.__gsheet_client.download_spreadsheet(spreadsheet_id=self.__gsheet_id, sheet_name=self.account,
                                                       file_path=file_path)
-        df = pd.read_csv(file_name)
-        df.fillna('', inplace=True)
-        if user_email:
-            df.set_index('_Email', inplace=True)
-            user = username.split('@')[0]
+        if os.path.exists(file_name):
+            df = pd.read_csv(file_name)
+            df.fillna('', inplace=True)
+            if user_email:
+                df.set_index('_Email', inplace=True)
+                user = username.split('@')[0]
+            else:
+                df.set_index('User', inplace=True)
+                user = username.split('_')[1].split('@')[0]
+            try:
+                tags = dict(df.loc[username])
+            except KeyError:
+                tags = {}
+            tags['User'] = user
         else:
-            df.set_index('User', inplace=True)
-            user = username.split('_')[1].split('@')[0]
-        tags = dict(df.loc[username])
-        tags['User'] = user
+            tags = {}
+            if '@redhat.com' in username:
+                tags = {'User': username.split('_')[-1].split('@')[0]}
         for key in list(tags.keys()):
             if key.startswith('_'):
                 tags.pop(key)
-        ldap_data = self.__ldap.get_user_details(user_name=tags['User'])
-        if ldap_data:
-            tags['Owner'] = ldap_data['FullName']
-            tags['Manager'] = ldap_data['managerName']
+        if tags:
+            ldap_data = self.__ldap.get_user_details(user_name=tags['User'])
+            if ldap_data:
+                tags['Owner'] = ldap_data['FullName']
+                tags['Manager'] = ldap_data['managerName']
         return self.__organise_user_tags(tags)
 
     @retry(exceptions=Exception, tries=RETRIES, delay=DELAY)
@@ -116,12 +126,14 @@ def get_monthly_invoices(self, month: int, year: int):
             }
         }
         invoice_mask = "mask[id, closedDate, typeCode, createDate]"
-        invoice_list = self.__sl_client.call('SoftLayer_Account', 'getInvoices', mask=invoice_mask, filter=_filter, iter=True)
+        invoice_list = self.__sl_client.call('SoftLayer_Account', 'getInvoices', mask=invoice_mask, filter=_filter,
+                                             iter=True)
         invoice_data = {}
         for invoice in invoice_list:
             if invoice.get('typeCode') == 'RECURRING':
                 invoice_item_mask = f"""mask[id, createDate, recurringFee, parentId, categoryCode, description, hostName, domainName, invoiceId, resourceTableId, productItemId]"""
-                invoice_items = self.__sl_client.call('SoftLayer_Billing_Invoice', 'getItems', id=invoice.get('id'), iter=True, mask=invoice_item_mask)
+                invoice_items = self.__sl_client.call('SoftLayer_Billing_Invoice', 'getItems', id=invoice.get('id'),
+                                                      iter=True, mask=invoice_item_mask)
                 invoice_data[invoice.get('id')] = invoice_items
         return invoice_data
 
@@ -168,7 +180,8 @@ def get_invoice_data(self, month: int, year: int):
                 else:
                     parent_id_data[item_id] = parent_id_data.get(item_id, 0) + recurring_fee
                 if username:
-                    description_id_data.setdefault(parent_id, set()).add(f'{username[0]}-{"-".join(description.split()[:2])}')
+                    description_id_data.setdefault(parent_id, set()).add(
+                        f'{username[0]}-{"-".join(description.split()[:2])}')
         for parent_id, username in description_id_data.items():
             hostname_data[parent_id] = list(username)[0]
         combine_invoice_data = {}
@@ -200,5 +213,7 @@ def get_daily_usage(self, month: int, year: int):
         @return:
         """
         billing_month = str(year) + '-' + str(month)  # yyyy-mm
-        account_summary = self.__service_client.get_account_summary(account_id=self.__account_id, billingmonth=billing_month, timeout=self.DELAY).get_result()
+        account_summary = self.__service_client.get_account_summary(account_id=self.__account_id,
+                                                                    billingmonth=billing_month,
+                                                                    timeout=self.DELAY).get_result()
         return account_summary

cloud_governance/common/clouds/ibm/account/ibm_authenticator.py

@@ -14,5 +14,9 @@ class IBMAuthenticator:
     def __init__(self):
         logging.disable(logging.DEBUG)
         self.env_config = environment_variables
-        self.__api_key = self.env_config.IBM_CLOUD_API_KEY
+        self.account_id = self.env_config.IBM_ACCOUNT_ID
+        if hasattr(self.env_config, 'IBM_CLOUD_API_KEY'):
+            self.__api_key = self.env_config.IBM_CLOUD_API_KEY
+        else:
+            self.__api_key = self.env_config.environment_variables_dict.get('IBM_CLOUD_API_KEY')
         self.iam_authenticator = IAMAuthenticator(self.__api_key)

cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py

@@ -28,8 +28,13 @@ def update_tags(self, resources_crn: list, tags: list):
                                 for i in range(0, len(resources_list), self.BATCH_SIZE)]
         success = 0
         errors = []
+        tag_names = []
+        for tag in tags:
+            key, value = tag.split(":")
+            tag_names.append(f'{key.strip()}:{value.strip()}')
+        logger.info(f"Tagging {len(resources_crn)} resources.")
         for resource_batch in resources_batch_list:
-            responses = self.__tag_service.attach_tag(resources=resource_batch, tag_names=tags) \
+            responses = self.__tag_service.attach_tag(resources=resource_batch, tag_names=tag_names) \
                 .get_result()['results']
             for resource in responses:
                 if resource['is_error']:

cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py

@@ -1,4 +1,6 @@
+import time
 from functools import wraps
+from urllib.parse import urlparse, parse_qs
 
 import ibm_vpc
 from typing import Callable
@@ -12,11 +14,22 @@ def wrapper(*args, **kwargs):
         vpc_obj = VpcInfraOperations()
         regions = vpc_obj.get_regions()
         resources_list = {}
+        exec_func = getattr(vpc_obj, func(*args, **kwargs), None)
         for region in regions:
             region_name = region.get('name')
             if region['status'] == 'available':
                 vpc_obj.set_service_url(region_name)
-                resources_list[region_name] = func(*args, **kwargs)
+                result = exec_func() if exec_func else []
+                if result:
+                    owned_resources = []
+                    for resource in result:
+                        if 'crn' in resource:
+                            if vpc_obj.account_id in resource['crn']:
+                                owned_resources.append(resource)
+                        else:
+                            owned_resources.append(resource)
+                    if owned_resources:
+                        resources_list[region_name] = owned_resources
         return resources_list
 
     return wrapper
@@ -50,7 +63,7 @@ def set_service_url(self, region_name: str):
         service_url = self.REGION_SERVICE_URL % region_name
         self.__client.set_service_url(service_url)
 
-    def iter_next_resources(self, exec_func: Callable, resource_name: str, region_name: str = None):
+    def iter_next_resources(self, exec_func: Callable, resource_name: str, region_name: str = None, **kwargs):
         """
         This method .
         :param region_name:
@@ -60,13 +73,20 @@ def iter_next_resources(self, exec_func: Callable, resource_name: str, region_na
         """
         if region_name:
             self.set_service_url(region_name)
-        response = exec_func().get_result()
+        response = exec_func(**kwargs).get_result()
         resources = response[resource_name]
+        count = 1
         while response.get('next'):
-            href = response['next']['href']
-            start = href.split('&')[-1].split('=')[-1]
-            response = exec_func(start=start).get_result()
-            resources.extend(response[resource_name])
+            parsed_url = urlparse(response['next']['href'])
+            params = parse_qs(parsed_url.query)
+            if params and 'start' in params:
+                start = params['start'][0]
+                response = exec_func(start=start, **kwargs).get_result()
+                resources.extend(response[resource_name])
+            count += 1
+            if count == 5:
+                time.sleep(30)
+                count = 0
         return resources
 
     def get_instances(self, region_name: str = None):
@@ -78,6 +98,24 @@ def get_instances(self, region_name: str = None):
         return self.iter_next_resources(exec_func=self.__client.list_instances,
                                         resource_name='instances', region_name=region_name)
 
+    def get_images(self, region_name: str = None):
+        """
+        This method lists available images in one region, default 'us-south'
+        :param region_name:
+        :return:
+        """
+        return self.iter_next_resources(exec_func=self.__client.list_images, resource_name='images',
+                                        region_name=region_name, status='available')
+
+    def get_placement_groups(self, region_name: str = None):
+        """
+        This method returns available placement groups in one region, default 'us-south'
+        :param region_name:
+        :return:
+        """
+        return self.iter_next_resources(exec_func=self.__client.list_placement_groups,
+                                        resource_name='placement_groups', region_name=region_name)
+
     def get_volumes(self, region_name: str = None):
         """
         This method lists available volumes.
@@ -149,74 +187,107 @@ def get_load_balancers(self, region_name: str = None):
         return self.iter_next_resources(exec_func=self.__client.list_load_balancers,
                                         resource_name='load_balancers', region_name=region_name)
 
+    def get_baremetal_servers(self, region_name: str = None):
+        """
+        This method lists available baremetals
+        :param region_name:
+        :return:
+        """
+        return self.iter_next_resources(exec_func=self.__client.list_bare_metal_servers,
+                                        resource_name='bare_metal_servers', region_name=region_name)
+
     @region_wrapper
     def get_all_instances(self):
         """
         This method lists all available instances.
         :return:
         """
-        return self.get_instances()
+        return "get_instances"
 
     @region_wrapper
     def get_all_volumes(self):
         """
         This method lists all available volumes.
         :return:
         """
-        return self.get_volumes()
+        return "get_volumes"
 
     @region_wrapper
     def get_all_vpcs(self):
         """
         This method lists all available vpc's.
         :return:
         """
-        return self.get_vpcs()
+        return "get_vpcs"
 
     @region_wrapper
     def get_all_floating_ips(self):
         """
         This method lists all floating ips.
         :return:
         """
-        return self.get_floating_ips()
+        return "get_floating_ips"
 
     @region_wrapper
     def get_all_virtual_network_interfaces(self):
         """
         This method lists all available virtual network interfaces.
         :return:
         """
-        return self.get_virtual_network_interfaces()
+        return "get_virtual_network_interfaces"
 
     @region_wrapper
     def get_all_security_groups(self):
         """
         This method lists all available security_groups
         :return:
         """
-        return self.get_security_groups()
+        return "get_security_groups"
 
     @region_wrapper
     def get_all_public_gateways(self):
         """
         This method lists all available public_gateways
         :return:
         """
-        return self.get_public_gateways()
+        return "get_public_gateways"
 
     @region_wrapper
     def get_all_vpc_endpoint_gateways(self):
         """
         This method lists all available vpc endpoint gateways
         :return:
         """
-        return self.get_vpc_endpoint_gateways()
+        return "get_vpc_endpoint_gateways"
 
     @region_wrapper
     def get_all_load_balancers(self):
         """
         This method lists all available load balancers.
         :return:
         """
-        return self.get_load_balancers()
+        return "get_load_balancers"
+
+    @region_wrapper
+    def get_all_baremetal_servers(self):
+        """
+        This method lists all available baremetals.
+        :return:
+        """
+        return "get_baremetal_servers"
+
+    @region_wrapper
+    def get_all_placement_groups(self):
+        """
+        This method lists all available placement groups.
+        :return:
+        """
+        return "get_placement_groups"
+
+    @region_wrapper
+    def get_all_images(self):
+        """
+        This method lists all available images.
+        :return:
+        """
+        return "get_images"

cloud_governance/common/google_drive/google_drive_operations.py

@@ -43,7 +43,8 @@ def create_work_sheet(self, gsheet_id: str, sheet_name: str):
                         }
                     }]}
                 if self.__service:
-                    self.__service.spreadsheets().batchUpdate(spreadsheetId=gsheet_id, body=create_worksheet_meta_data).execute()
+                    self.__service.spreadsheets().batchUpdate(spreadsheetId=gsheet_id,
+                                                              body=create_worksheet_meta_data).execute()
                     logger.info(f'{sheet_name} worksheet created')
             else:
                 logger.info(f'{sheet_name} Worksheet Already present')
@@ -62,7 +63,8 @@ def download_spreadsheet(self, spreadsheet_id: str, sheet_name: str, file_path:
         """
         if self.__service:
             try:
-                result = self.__service.spreadsheets().values().get(spreadsheetId=spreadsheet_id, range=sheet_name).execute()
+                result = self.__service.spreadsheets().values().get(spreadsheetId=spreadsheet_id,
+                                                                    range=sheet_name).execute()
                 file_name = f'{sheet_name}.csv'
                 output_file = os.path.join(file_path, file_name)
                 if result.get('values'):
@@ -104,7 +106,8 @@ def find_sheet_id_by_name(self, sheet_name: str, spreadsheet_id: str):
         """
         if self.__service:
             sheets_with_properties = self.__service.spreadsheets().get(spreadsheetId=spreadsheet_id,
-                                                                       fields='sheets.properties').execute().get('sheets')
+                                                                       fields='sheets.properties').execute().get(
+                'sheets')
             for sheet in sheets_with_properties:
                 if 'title' in sheet['properties'].keys():
                     if sheet['properties']['title'] == sheet_name:
@@ -126,7 +129,8 @@ def delete_rows(self, spreadsheet_id: str, sheet_name: str, row_number: int):
                     {
                         "deleteDimension": {
                             "range": {
-                                "sheetId": self.find_sheet_id_by_name(sheet_name=sheet_name, spreadsheet_id=spreadsheet_id),
+                                "sheetId": self.find_sheet_id_by_name(sheet_name=sheet_name,
+                                                                      spreadsheet_id=spreadsheet_id),
                                 "dimension": "ROWS",
                                 "startIndex": row_number,
                                 "endIndex": row_number + 1

cloud_governance/main/environment_variables.py

@@ -153,6 +153,7 @@ def __init__(self):
         if (self._environment_variables_dict['USAGE_REPORTS_APIKEY'] or
                 self._environment_variables_dict['IBM_CLOUD_API_KEY'] or
                 hasattr(self, "IBM_CLOUD_API_KEY")):
+            self.IBM_CLOUD_API_KEY = self._environment_variables_dict['IBM_CLOUD_API_KEY']
             self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'IBM'
         self._environment_variables_dict['month'] = EnvironmentVariables.get_env('month', '')
         self._environment_variables_dict['year'] = EnvironmentVariables.get_env('year', '')