From 0a981ffb7e44c44fbf525ca68d94d101352d4162 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti Date: Fri, 15 Nov 2024 23:21:07 +0530 Subject: [PATCH] Added the IBM resources to tagging --- .../common/clouds/ibm/account/ibm_account.py | 51 ++++++--- .../clouds/ibm/account/ibm_authenticator.py | 6 +- .../ibm/tagging/global_tagging_operations.py | 7 +- .../clouds/ibm/vpc/vpc_infra_operations.py | 103 +++++++++++++++--- .../google_drive/google_drive_operations.py | 12 +- .../main/environment_variables.py | 1 + cloud_governance/policy/ibm/tag_baremetal.py | 26 +++-- cloud_governance/policy/ibm/tag_resources.py | 37 ++++++- cloud_governance/policy/ibm/tag_vm.py | 16 ++- jenkins/clouds/ibm/hourly/tagging/Jenkinsfile | 2 + jenkins/clouds/ibm/hourly/tagging/tagging.py | 8 +- requirements.txt | 1 + setup.py | 1 + .../global_tagging/test_ibm_global_tagging.py | 1 + .../ibm/vpc/test_vpc_infra_operations.py | 1 + .../policy/ibm/test_tag_resources.py | 1 + 16 files changed, 214 insertions(+), 60 deletions(-) diff --git a/cloud_governance/common/clouds/ibm/account/ibm_account.py b/cloud_governance/common/clouds/ibm/account/ibm_account.py index 61b25d77..9bd390e2 100644 --- a/cloud_governance/common/clouds/ibm/account/ibm_account.py +++ b/cloud_governance/common/clouds/ibm/account/ibm_account.py @@ -33,7 +33,8 @@ def __init__(self): self.__API_KEY = self.__environment_variables_dict.get('IBM_API_KEY', '') try: if self.__API_KEY and self.__API_USERNAME: - self.__sl_client = SoftLayer.create_client_from_env(username=self.__API_USERNAME, api_key=self.__API_KEY, timeout=self.DELAY) + self.__sl_client = SoftLayer.create_client_from_env(username=self.__API_USERNAME, + api_key=self.__API_KEY, timeout=self.DELAY) self.short_account_id = str(self.__API_USERNAME.split('_')[0]) if self.__environment_variables_dict.get('USAGE_REPORTS_APIKEY'): self.__service_client = UsageReportsV4.new_instance() @@ -81,23 +82,32 @@ def get_user_tags_from_gsheet(self, username: str, user_email: str = '', file_pa if not os.path.exists(file_name): self.__gsheet_client.download_spreadsheet(spreadsheet_id=self.__gsheet_id, sheet_name=self.account, file_path=file_path) - df = pd.read_csv(file_name) - df.fillna('', inplace=True) - if user_email: - df.set_index('_Email', inplace=True) - user = username.split('@')[0] + if os.path.exists(file_name): + df = pd.read_csv(file_name) + df.fillna('', inplace=True) + if user_email: + df.set_index('_Email', inplace=True) + user = username.split('@')[0] + else: + df.set_index('User', inplace=True) + user = username.split('_')[1].split('@')[0] + try: + tags = dict(df.loc[username]) + except KeyError: + tags = {} + tags['User'] = user else: - df.set_index('User', inplace=True) - user = username.split('_')[1].split('@')[0] - tags = dict(df.loc[username]) - tags['User'] = user + tags = {} + if '@redhat.com' in username: + tags = {'User': username.split('_')[-1].split('@')[0]} for key in list(tags.keys()): if key.startswith('_'): tags.pop(key) - ldap_data = self.__ldap.get_user_details(user_name=tags['User']) - if ldap_data: - tags['Owner'] = ldap_data['FullName'] - tags['Manager'] = ldap_data['managerName'] + if tags: + ldap_data = self.__ldap.get_user_details(user_name=tags['User']) + if ldap_data: + tags['Owner'] = ldap_data['FullName'] + tags['Manager'] = ldap_data['managerName'] return self.__organise_user_tags(tags) @retry(exceptions=Exception, tries=RETRIES, delay=DELAY) @@ -116,12 +126,14 @@ def get_monthly_invoices(self, month: int, year: int): } } invoice_mask = "mask[id, closedDate, typeCode, createDate]" - invoice_list = self.__sl_client.call('SoftLayer_Account', 'getInvoices', mask=invoice_mask, filter=_filter, iter=True) + invoice_list = self.__sl_client.call('SoftLayer_Account', 'getInvoices', mask=invoice_mask, filter=_filter, + iter=True) invoice_data = {} for invoice in invoice_list: if invoice.get('typeCode') == 'RECURRING': invoice_item_mask = f"""mask[id, createDate, recurringFee, parentId, categoryCode, description, hostName, domainName, invoiceId, resourceTableId, productItemId]""" - invoice_items = self.__sl_client.call('SoftLayer_Billing_Invoice', 'getItems', id=invoice.get('id'), iter=True, mask=invoice_item_mask) + invoice_items = self.__sl_client.call('SoftLayer_Billing_Invoice', 'getItems', id=invoice.get('id'), + iter=True, mask=invoice_item_mask) invoice_data[invoice.get('id')] = invoice_items return invoice_data @@ -168,7 +180,8 @@ def get_invoice_data(self, month: int, year: int): else: parent_id_data[item_id] = parent_id_data.get(item_id, 0) + recurring_fee if username: - description_id_data.setdefault(parent_id, set()).add(f'{username[0]}-{"-".join(description.split()[:2])}') + description_id_data.setdefault(parent_id, set()).add( + f'{username[0]}-{"-".join(description.split()[:2])}') for parent_id, username in description_id_data.items(): hostname_data[parent_id] = list(username)[0] combine_invoice_data = {} @@ -200,5 +213,7 @@ def get_daily_usage(self, month: int, year: int): @return: """ billing_month = str(year) + '-' + str(month) # yyyy-mm - account_summary = self.__service_client.get_account_summary(account_id=self.__account_id, billingmonth=billing_month, timeout=self.DELAY).get_result() + account_summary = self.__service_client.get_account_summary(account_id=self.__account_id, + billingmonth=billing_month, + timeout=self.DELAY).get_result() return account_summary diff --git a/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py b/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py index 46d72e0b..e933284b 100644 --- a/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py +++ b/cloud_governance/common/clouds/ibm/account/ibm_authenticator.py @@ -14,5 +14,9 @@ class IBMAuthenticator: def __init__(self): logging.disable(logging.DEBUG) self.env_config = environment_variables - self.__api_key = self.env_config.IBM_CLOUD_API_KEY + self.account_id = self.env_config.IBM_ACCOUNT_ID + if hasattr(self.env_config, 'IBM_CLOUD_API_KEY'): + self.__api_key = self.env_config.IBM_CLOUD_API_KEY + else: + self.__api_key = self.env_config.environment_variables_dict.get('IBM_CLOUD_API_KEY') self.iam_authenticator = IAMAuthenticator(self.__api_key) diff --git a/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py b/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py index 8bccc2b6..122a529e 100644 --- a/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py +++ b/cloud_governance/common/clouds/ibm/tagging/global_tagging_operations.py @@ -28,8 +28,13 @@ def update_tags(self, resources_crn: list, tags: list): for i in range(0, len(resources_list), self.BATCH_SIZE)] success = 0 errors = [] + tag_names = [] + for tag in tags: + key, value = tag.split(":") + tag_names.append(f'{key.strip()}:{value.strip()}') + logger.info(f"Tagging {len(resources_crn)} resources.") for resource_batch in resources_batch_list: - responses = self.__tag_service.attach_tag(resources=resource_batch, tag_names=tags) \ + responses = self.__tag_service.attach_tag(resources=resource_batch, tag_names=tag_names) \ .get_result()['results'] for resource in responses: if resource['is_error']: diff --git a/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py b/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py index a1411581..86269a90 100644 --- a/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py +++ b/cloud_governance/common/clouds/ibm/vpc/vpc_infra_operations.py @@ -1,4 +1,6 @@ +import time from functools import wraps +from urllib.parse import urlparse, parse_qs import ibm_vpc from typing import Callable @@ -12,11 +14,22 @@ def wrapper(*args, **kwargs): vpc_obj = VpcInfraOperations() regions = vpc_obj.get_regions() resources_list = {} + exec_func = getattr(vpc_obj, func(*args, **kwargs), None) for region in regions: region_name = region.get('name') if region['status'] == 'available': vpc_obj.set_service_url(region_name) - resources_list[region_name] = func(*args, **kwargs) + result = exec_func() if exec_func else [] + if result: + owned_resources = [] + for resource in result: + if 'crn' in resource: + if vpc_obj.account_id in resource['crn']: + owned_resources.append(resource) + else: + owned_resources.append(resource) + if owned_resources: + resources_list[region_name] = owned_resources return resources_list return wrapper @@ -50,7 +63,7 @@ def set_service_url(self, region_name: str): service_url = self.REGION_SERVICE_URL % region_name self.__client.set_service_url(service_url) - def iter_next_resources(self, exec_func: Callable, resource_name: str, region_name: str = None): + def iter_next_resources(self, exec_func: Callable, resource_name: str, region_name: str = None, **kwargs): """ This method . :param region_name: @@ -60,13 +73,20 @@ def iter_next_resources(self, exec_func: Callable, resource_name: str, region_na """ if region_name: self.set_service_url(region_name) - response = exec_func().get_result() + response = exec_func(**kwargs).get_result() resources = response[resource_name] + count = 1 while response.get('next'): - href = response['next']['href'] - start = href.split('&')[-1].split('=')[-1] - response = exec_func(start=start).get_result() - resources.extend(response[resource_name]) + parsed_url = urlparse(response['next']['href']) + params = parse_qs(parsed_url.query) + if params and 'start' in params: + start = params['start'][0] + response = exec_func(start=start, **kwargs).get_result() + resources.extend(response[resource_name]) + count += 1 + if count == 5: + time.sleep(30) + count = 0 return resources def get_instances(self, region_name: str = None): @@ -78,6 +98,24 @@ def get_instances(self, region_name: str = None): return self.iter_next_resources(exec_func=self.__client.list_instances, resource_name='instances', region_name=region_name) + def get_images(self, region_name: str = None): + """ + This method lists available images in one region, default 'us-south' + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_images, resource_name='images', + region_name=region_name, status='available') + + def get_placement_groups(self, region_name: str = None): + """ + This method returns available placement groups in one region, default 'us-south' + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_placement_groups, + resource_name='placement_groups', region_name=region_name) + def get_volumes(self, region_name: str = None): """ This method lists available volumes. @@ -149,13 +187,22 @@ def get_load_balancers(self, region_name: str = None): return self.iter_next_resources(exec_func=self.__client.list_load_balancers, resource_name='load_balancers', region_name=region_name) + def get_baremetal_servers(self, region_name: str = None): + """ + This method lists available baremetals + :param region_name: + :return: + """ + return self.iter_next_resources(exec_func=self.__client.list_bare_metal_servers, + resource_name='bare_metal_servers', region_name=region_name) + @region_wrapper def get_all_instances(self): """ This method lists all available instances. :return: """ - return self.get_instances() + return "get_instances" @region_wrapper def get_all_volumes(self): @@ -163,7 +210,7 @@ def get_all_volumes(self): This method lists all available volumes. :return: """ - return self.get_volumes() + return "get_volumes" @region_wrapper def get_all_vpcs(self): @@ -171,7 +218,7 @@ def get_all_vpcs(self): This method lists all available vpc's. :return: """ - return self.get_vpcs() + return "get_vpcs" @region_wrapper def get_all_floating_ips(self): @@ -179,7 +226,7 @@ def get_all_floating_ips(self): This method lists all floating ips. :return: """ - return self.get_floating_ips() + return "get_floating_ips" @region_wrapper def get_all_virtual_network_interfaces(self): @@ -187,7 +234,7 @@ def get_all_virtual_network_interfaces(self): This method lists all available virtual network interfaces. :return: """ - return self.get_virtual_network_interfaces() + return "get_virtual_network_interfaces" @region_wrapper def get_all_security_groups(self): @@ -195,7 +242,7 @@ def get_all_security_groups(self): This method lists all available security_groups :return: """ - return self.get_security_groups() + return "get_security_groups" @region_wrapper def get_all_public_gateways(self): @@ -203,7 +250,7 @@ def get_all_public_gateways(self): This method lists all available public_gateways :return: """ - return self.get_public_gateways() + return "get_public_gateways" @region_wrapper def get_all_vpc_endpoint_gateways(self): @@ -211,7 +258,7 @@ def get_all_vpc_endpoint_gateways(self): This method lists all available vpc endpoint gateways :return: """ - return self.get_vpc_endpoint_gateways() + return "get_vpc_endpoint_gateways" @region_wrapper def get_all_load_balancers(self): @@ -219,4 +266,28 @@ def get_all_load_balancers(self): This method lists all available load balancers. :return: """ - return self.get_load_balancers() + return "get_load_balancers" + + @region_wrapper + def get_all_baremetal_servers(self): + """ + This method lists all available baremetals. + :return: + """ + return "get_baremetal_servers" + + @region_wrapper + def get_all_placement_groups(self): + """ + This method lists all available placement groups. + :return: + """ + return "get_placement_groups" + + @region_wrapper + def get_all_images(self): + """ + This method lists all available images. + :return: + """ + return "get_images" diff --git a/cloud_governance/common/google_drive/google_drive_operations.py b/cloud_governance/common/google_drive/google_drive_operations.py index a0e37560..a5e0f796 100644 --- a/cloud_governance/common/google_drive/google_drive_operations.py +++ b/cloud_governance/common/google_drive/google_drive_operations.py @@ -43,7 +43,8 @@ def create_work_sheet(self, gsheet_id: str, sheet_name: str): } }]} if self.__service: - self.__service.spreadsheets().batchUpdate(spreadsheetId=gsheet_id, body=create_worksheet_meta_data).execute() + self.__service.spreadsheets().batchUpdate(spreadsheetId=gsheet_id, + body=create_worksheet_meta_data).execute() logger.info(f'{sheet_name} worksheet created') else: logger.info(f'{sheet_name} Worksheet Already present') @@ -62,7 +63,8 @@ def download_spreadsheet(self, spreadsheet_id: str, sheet_name: str, file_path: """ if self.__service: try: - result = self.__service.spreadsheets().values().get(spreadsheetId=spreadsheet_id, range=sheet_name).execute() + result = self.__service.spreadsheets().values().get(spreadsheetId=spreadsheet_id, + range=sheet_name).execute() file_name = f'{sheet_name}.csv' output_file = os.path.join(file_path, file_name) if result.get('values'): @@ -104,7 +106,8 @@ def find_sheet_id_by_name(self, sheet_name: str, spreadsheet_id: str): """ if self.__service: sheets_with_properties = self.__service.spreadsheets().get(spreadsheetId=spreadsheet_id, - fields='sheets.properties').execute().get('sheets') + fields='sheets.properties').execute().get( + 'sheets') for sheet in sheets_with_properties: if 'title' in sheet['properties'].keys(): if sheet['properties']['title'] == sheet_name: @@ -126,7 +129,8 @@ def delete_rows(self, spreadsheet_id: str, sheet_name: str, row_number: int): { "deleteDimension": { "range": { - "sheetId": self.find_sheet_id_by_name(sheet_name=sheet_name, spreadsheet_id=spreadsheet_id), + "sheetId": self.find_sheet_id_by_name(sheet_name=sheet_name, + spreadsheet_id=spreadsheet_id), "dimension": "ROWS", "startIndex": row_number, "endIndex": row_number + 1 diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index 6eaea4d2..578115c3 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -153,6 +153,7 @@ def __init__(self): if (self._environment_variables_dict['USAGE_REPORTS_APIKEY'] or self._environment_variables_dict['IBM_CLOUD_API_KEY'] or hasattr(self, "IBM_CLOUD_API_KEY")): + self.IBM_CLOUD_API_KEY = self._environment_variables_dict['IBM_CLOUD_API_KEY'] self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'IBM' self._environment_variables_dict['month'] = EnvironmentVariables.get_env('month', '') self._environment_variables_dict['year'] = EnvironmentVariables.get_env('year', '') diff --git a/cloud_governance/policy/ibm/tag_baremetal.py b/cloud_governance/policy/ibm/tag_baremetal.py index 6e1ce307..e3b7335e 100644 --- a/cloud_governance/policy/ibm/tag_baremetal.py +++ b/cloud_governance/policy/ibm/tag_baremetal.py @@ -1,4 +1,3 @@ - from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp from cloud_governance.policy.policy_operations.ibm.tagging.tagging_operations import TaggingOperations @@ -20,7 +19,8 @@ def get_hardware_username(self, hardware_id: str): """ hardware_data = self._classic_operations.get_hardware_data(hardware_id=str(hardware_id)) if hardware_data: - return hardware_data.get('billingItem').get('orderItem').get('order').get('userRecord').get('username'), f'{hardware_data.get("hostname")}.{hardware_data.get("domain")}' + return hardware_data.get('billingItem').get('orderItem').get('order').get('userRecord').get( + 'username'), f'{hardware_data.get("hostname")}.{hardware_data.get("domain")}' return '', '' def tag_remove_hardware(self, user_tags: list, hardware_tags: list, hardware_id: str, hardware_name: str): @@ -37,11 +37,15 @@ def tag_remove_hardware(self, user_tags: list, hardware_tags: list, hardware_id: remove_hardware_tags = [self._tag_remove_name] if remove_hardware_tags: try: - response = self.softlayer_operation(softlayer_name='SoftLayer_Hardware_Server', softlayer_method='removeTags', resource_id=hardware_id, tags=','.join(remove_hardware_tags)) + response = self.softlayer_operation(softlayer_name='SoftLayer_Hardware_Server', + softlayer_method='removeTags', resource_id=hardware_id, + tags=','.join(remove_hardware_tags)) if response: - logger.info(f'Tags are Removed to the hardware: {hardware_id} - {hardware_name} : count: {len(remove_hardware_tags)} : {remove_hardware_tags}') + logger.info( + f'Tags are Removed to the hardware: {hardware_id} - {hardware_name} : count: {len(remove_hardware_tags)} : {remove_hardware_tags}') else: - logger.info(f'Tags are not Removed to the Hardware: {hardware_id} - {hardware_name}, something might fail') + logger.info( + f'Tags are not Removed to the Hardware: {hardware_id} - {hardware_name}, something might fail') except Exception as err: logger.info(f'{err}') return remove_hardware_tags @@ -57,18 +61,22 @@ def tag_update_hardware(self, user_tags: list, hardware_tags: list, hardware_id: """ add_hardware_tags = [] if self._tag_custom: - add_hardware_tags.extend(self._tag_custom) + user_tags.extend(self._tag_custom) add_hardware_tags.extend(self._filter_common_tags(user_tags=user_tags, resource_tags=hardware_tags)) if add_hardware_tags: if self._tag_operation == 'update': add_hardware_tags.extend(hardware_tags) add_hardware_tags = list(set(add_hardware_tags)) try: - response = self.softlayer_operation(softlayer_name='SoftLayer_Hardware_Server', softlayer_method='setTags', resource_id=hardware_id, tags=','.join(add_hardware_tags)) + response = self.softlayer_operation(softlayer_name='SoftLayer_Hardware_Server', + softlayer_method='setTags', resource_id=hardware_id, + tags=','.join(add_hardware_tags)) if response: - logger.info(f'Tags are added to the hardware: {hardware_id} - {hardware_name} : count: {len(add_hardware_tags)} : {add_hardware_tags}') + logger.info( + f'Tags are added to the hardware: {hardware_id} - {hardware_name} : count: {len(add_hardware_tags)} : {add_hardware_tags}') else: - logger.info(f'Tags are not added to the hardware: {hardware_id} - {hardware_name}, something might fail') + logger.info( + f'Tags are not added to the hardware: {hardware_id} - {hardware_name}, something might fail') except Exception as err: logger.info(f'{err}') return add_hardware_tags diff --git a/cloud_governance/policy/ibm/tag_resources.py b/cloud_governance/policy/ibm/tag_resources.py index 1a474537..f978588c 100644 --- a/cloud_governance/policy/ibm/tag_resources.py +++ b/cloud_governance/policy/ibm/tag_resources.py @@ -13,8 +13,9 @@ def wrapper(*args, **kwargs): resources_crn = [] resource_list = func(*args, **kwargs) for region, resources in resource_list.items(): - resources_crn.extend([resource.get('crn') for resource in resources]) - return resources_crn + resources_crn.extend( + [resource.get('crn') if isinstance(resource, dict) else resource for resource in resources]) + return list(set(resources_crn)) return wrapper @@ -45,6 +46,24 @@ def get_virtual_servers_crn(self): """ return self.vpc_infra_operations.get_all_instances() + @get_resources_wrapper + @logger_time_stamp + def get_images_crn(self): + """ + This method returns all virtual server crn's + :return: + """ + return self.vpc_infra_operations.get_all_images() + + @get_resources_wrapper + @logger_time_stamp + def get_placement_groups_crn(self): + """ + This method returns all placement group crn's + :return: + """ + return self.vpc_infra_operations.get_all_placement_groups() + @get_resources_wrapper @logger_time_stamp def get_volumes_crn(self): @@ -126,6 +145,15 @@ def get_load_balancers_crn(self): """ return self.vpc_infra_operations.get_all_load_balancers() + @get_resources_wrapper + @logger_time_stamp + def get_baremetal_servers_crn(self): + """ + This method returns all baremetals crn's + :return: + """ + return self.vpc_infra_operations.get_all_baremetal_servers() + @logger_time_stamp def tag_all_vpc_resources(self): """ @@ -139,6 +167,7 @@ def tag_all_vpc_resources(self): tags_list = self.__ibm_custom_tags_list.split(',') vpc_resources = [ "virtual_servers", + "placement_groups", "volumes", "floating_ips", "vpcs", @@ -147,7 +176,9 @@ def tag_all_vpc_resources(self): "public_gateways", "vpc_endpoint_gateways", "load_balancers", - "schematics_workspaces" + "schematics_workspaces", + "baremetal_servers", + "images" ] if self.resource_to_tag and self.resource_to_tag in vpc_resources: vpc_resources = [self.resource_to_tag] diff --git a/cloud_governance/policy/ibm/tag_vm.py b/cloud_governance/policy/ibm/tag_vm.py index b66b9987..07ea9548 100644 --- a/cloud_governance/policy/ibm/tag_vm.py +++ b/cloud_governance/policy/ibm/tag_vm.py @@ -18,7 +18,8 @@ def get_virtual_machine_username(self, vm_id: str): @return: """ vm_data = self._classic_operations.get_virtual_machine_data(vm_id=str(vm_id)) - return vm_data.get('billingItem').get('orderItem').get('order').get('userRecord').get('username'), f'{vm_data.get("hostname")}.{vm_data.get("domain")}' + return vm_data.get('billingItem').get('orderItem').get('order').get('userRecord').get( + 'username'), f'{vm_data.get("hostname")}.{vm_data.get("domain")}' def tag_update_virtual_machine(self, user_tags: list, vm_tags: list, vm_id: str, vm_name: str): """ @@ -31,7 +32,7 @@ def tag_update_virtual_machine(self, user_tags: list, vm_tags: list, vm_id: str, """ add_vm_tags = [] if self._tag_custom: - add_vm_tags.extend(self._tag_custom) + user_tags.extend(self._tag_custom) add_vm_tags.extend(self._filter_common_tags(user_tags=user_tags, resource_tags=vm_tags)) if add_vm_tags: tags = add_vm_tags @@ -39,7 +40,9 @@ def tag_update_virtual_machine(self, user_tags: list, vm_tags: list, vm_id: str, add_vm_tags.extend(vm_tags) add_vm_tags = list(set(add_vm_tags)) try: - response = self.softlayer_operation(softlayer_name='SoftLayer_Virtual_Guest', softlayer_method='setTags', tags=','.join(add_vm_tags), resource_id=vm_id) + response = self.softlayer_operation(softlayer_name='SoftLayer_Virtual_Guest', + softlayer_method='setTags', tags=','.join(add_vm_tags), + resource_id=vm_id) if response: logger.info(f'Tags are added to the vm: {vm_id} - {vm_name} : count: {len(tags)} : {tags}') else: @@ -62,9 +65,12 @@ def tag_remove_virtual_machine(self, user_tags: list, vm_tags: list, vm_id: str, remove_vm_tags = [self._tag_remove_name] if remove_vm_tags: try: - response = self.softlayer_operation(softlayer_name='SoftLayer_Virtual_Guest', softlayer_method='removeTags', tags=','.join(remove_vm_tags), resource_id=vm_id) + response = self.softlayer_operation(softlayer_name='SoftLayer_Virtual_Guest', + softlayer_method='removeTags', tags=','.join(remove_vm_tags), + resource_id=vm_id) if response: - logger.info(f'Tags are Removed to the vm: {vm_id} - {vm_name} : count: {len(remove_vm_tags)} : {remove_vm_tags}') + logger.info( + f'Tags are Removed to the vm: {vm_id} - {vm_name} : count: {len(remove_vm_tags)} : {remove_vm_tags}') else: logger.info(f'Tags are not Removed to the vm: {vm_id} - {vm_name}, something might fail') except Exception as err: diff --git a/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile b/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile index 2126fd05..a94f5fb3 100644 --- a/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile +++ b/jenkins/clouds/ibm/hourly/tagging/Jenkinsfile @@ -18,6 +18,8 @@ pipeline { LDAP_HOST_NAME = credentials('cloud-governance-ldap-host-name') IBM_CUSTOM_TAGS_LIST = credentials('IBM_CUSTOM_TAGS_LIST') IBM_CLOUD_API_KEY = credentials('IBM_CLOUD_API_KEY') + IBM_ACCOUNT_ID_PERFORMANCE_SCALE = credentials('cloud-governance-ibm-account-id-performance-scale') + account = "IBM-PERF" contact1 = "ebattat@redhat.com" contact2 = "athiruma@redhat.com" diff --git a/jenkins/clouds/ibm/hourly/tagging/tagging.py b/jenkins/clouds/ibm/hourly/tagging/tagging.py index 69ed6adc..b4f4a7e0 100644 --- a/jenkins/clouds/ibm/hourly/tagging/tagging.py +++ b/jenkins/clouds/ibm/hourly/tagging/tagging.py @@ -9,6 +9,7 @@ IBM_CLOUD_API_KEY = os.environ['IBM_CLOUD_API_KEY'] LOGS = os.environ.get('LOGS', 'logs') account = os.environ['account'] +IBM_ACCOUNT_ID_PERFORMANCE_SCALE = os.environ['IBM_ACCOUNT_ID_PERFORMANCE_SCALE'] QUAY_CLOUD_GOVERNANCE_REPOSITORY = os.environ.get('QUAY_CLOUD_GOVERNANCE_REPOSITORY', 'quay.io/cloud-governance/cloud-governance:latest') @@ -47,11 +48,11 @@ def get_podman_run_cmd(volume_mounts: list = None, **kwargs): run_cmd("echo Run IBM tag baremetal") volume_mounts_targets = [GOOGLE_APPLICATION_CREDENTIALS] - +tag_custom = [IBM_CUSTOM_TAGS_LIST] input_env_keys = {'account': account, 'LDAP_HOST_NAME': LDAP_HOST_NAME, 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, 'SPREADSHEET_ID': SPREADSHEET_ID, 'IBM_API_USERNAME': IBM_API_USERNAME, 'IBM_API_KEY': IBM_API_KEY, 'tag_operation': "update", - 'log_level': "INFO", 'policy': 'tag_baremetal', "PUBLIC_CLOUD_NAME": "IBM"} + 'log_level': "INFO", 'policy': 'tag_baremetal', "PUBLIC_CLOUD_NAME": "IBM", "tag_custom": tag_custom} baremetal_cmd = get_podman_run_cmd(volume_mounts=volume_mounts_targets, **input_env_keys) run_cmd(baremetal_cmd) @@ -66,5 +67,6 @@ def get_podman_run_cmd(volume_mounts: list = None, **kwargs): podman_run_cmd = get_podman_run_cmd(policy="tag_resources", account=account, IBM_CLOUD_API_KEY=IBM_CLOUD_API_KEY, IBM_CUSTOM_TAGS_LIST=IBM_CUSTOM_TAGS_LIST, - PUBLIC_CLOUD_NAME="IBM") + PUBLIC_CLOUD_NAME="IBM", + IBM_ACCOUNT_ID=IBM_ACCOUNT_ID_PERFORMANCE_SCALE) run_cmd(podman_run_cmd) diff --git a/requirements.txt b/requirements.txt index ab325533..39811706 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ google-auth-oauthlib==0.5.2 google-cloud-bigquery==3.5.0 google-cloud-billing==1.9.1 ibm-cloud-sdk-core==3.18.0 +ibm-cos-sdk==2.13.6 ibm-platform-services==0.27.0 ibm-schematics==1.1.0 ibm-vpc==0.21.0 diff --git a/setup.py b/setup.py index 9ceeca5f..f1b41c1d 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ 'google-cloud-bigquery==3.5.0', # google cloud cost 'google-cloud-billing==1.9.1', # google cloud cost 'ibm-cloud-sdk-core==3.18.0', + 'ibm-cos-sdk==2.13.6', 'ibm-platform-services==0.27.0', # IBM Usage reports 'ibm-schematics==1.1.0', 'ibm-vpc==0.21.0', diff --git a/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py b/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py index 53520d44..acf306da 100644 --- a/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py +++ b/tests/unittest/cloud_governance/common/clouds/ibm/global_tagging/test_ibm_global_tagging.py @@ -3,6 +3,7 @@ from tests.unittest.mocks.ibm.mock_ibm_global_tagging import mock_ibm_global_tagging environment_variables.IBM_CLOUD_API_KEY = 'mock_ibm_api_key' +environment_variables.IBM_ACCOUNT_ID = "test" @mock_ibm_global_tagging diff --git a/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py b/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py index ba67bacf..5143eff2 100644 --- a/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py +++ b/tests/unittest/cloud_governance/common/clouds/ibm/vpc/test_vpc_infra_operations.py @@ -3,6 +3,7 @@ from tests.unittest.mocks.ibm.mock_ibm_vpc import mock_ibm_vpc environment_variables.IBM_CLOUD_API_KEY = 'mock_ibm_api_key' +environment_variables.IBM_ACCOUNT_ID = "test" @mock_ibm_vpc diff --git a/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py b/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py index 6fd47b40..6a74b006 100644 --- a/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py +++ b/tests/unittest/cloud_governance/policy/ibm/test_tag_resources.py @@ -10,6 +10,7 @@ def test_tag_all_vpc_resources(): environment_variables.IBM_CLOUD_API_KEY = 'mock_ibm_api_key' environment_variables.RESOURCE_TO_TAG = 'virtual_servers' environment_variables.IBM_CUSTOM_TAGS_LIST = "cost-center: test" + environment_variables.IBM_ACCOUNT_ID = "test" tag_resources = TagResources() res = tag_resources.tag_all_vpc_resources() assert res.get('messages').get('virtual_servers')