diff --git a/cloud_governance/common/google_drive/gcp_operations.py b/cloud_governance/common/google_drive/gcp_operations.py new file mode 100644 index 00000000..2f426678 --- /dev/null +++ b/cloud_governance/common/google_drive/gcp_operations.py @@ -0,0 +1,26 @@ +import os +import tempfile + +import pandas as pd + +from cloud_governance.common.google_drive.google_drive_operations import GoogleDriveOperations +from cloud_governance.main.environment_variables import environment_variables + + +class GCPOperations: + + def __init__(self): + self.__environment_variables_dict = environment_variables.environment_variables_dict + self.__gsheet_operations = GoogleDriveOperations() + self.__gsheet_id = self.__environment_variables_dict.get('SPREADSHEET_ID', '') + + def get_accounts_sheet(self, sheet_name: str, dir_path: str = None): + with tempfile.TemporaryDirectory() as tmp_dir: + dir_path = dir_path if dir_path else tmp_dir + file_path = f'{dir_path}/{sheet_name}.csv' + if not os.path.exists(file_path): + self.__gsheet_operations.download_spreadsheet(spreadsheet_id=self.__gsheet_id, + sheet_name=sheet_name, + file_path=dir_path) + accounts_df = pd.read_csv(file_path) + return accounts_df diff --git a/cloud_governance/common/utils/api_requests.py b/cloud_governance/common/utils/api_requests.py new file mode 100644 index 00000000..400e3045 --- /dev/null +++ b/cloud_governance/common/utils/api_requests.py @@ -0,0 +1,29 @@ +import asyncio + +import aiohttp +import requests + +from cloud_governance.common.logger.init_logger import logger + + +class APIRequests: + + def __init__(self): + self.__loop = asyncio.new_event_loop() + + def get(self, url: str, **kwargs): + try: + response = requests.get(url, **kwargs) + if response.ok: + return response.json() + else: + return response.text + except Exception as err: + raise err + + def post(self, url: str, **kwargs): + try: + response = requests.post(url, **kwargs) + return response + except Exception as err: + raise err diff --git a/cloud_governance/common/utils/configs.py b/cloud_governance/common/utils/configs.py index f46e6d88..12900afa 100644 --- a/cloud_governance/common/utils/configs.py +++ b/cloud_governance/common/utils/configs.py @@ -1,3 +1,9 @@ LOOK_BACK_DAYS = 30 +MONTHS = 12 +DEFAULT_ROUND_DIGITS = 3 +PUBLIC_ACCOUNTS_COST_REPORTS = 'Accounts' + + +DATE_FORMAT = "%Y-%m-%d" diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index cf4d0483..1fd97c43 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -196,7 +196,7 @@ def __init__(self): self._environment_variables_dict['KERBEROS_USERS'] = literal_eval(EnvironmentVariables.get_env('KERBEROS_USERS', '[]')) self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval(EnvironmentVariables.get_env('POLICIES_TO_ALERT', '[]')) self._environment_variables_dict['ADMIN_MAIL_LIST'] = EnvironmentVariables.get_env('ADMIN_MAIL_LIST', '') - if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts']: + if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts', 'cloudability_cost_reports']: self._environment_variables_dict['COMMON_POLICIES'] = True # CRO -- Cloud Resource Orch self._environment_variables_dict['CLOUD_RESOURCE_ORCHESTRATION'] = EnvironmentVariables.get_boolean_from_environment('CLOUD_RESOURCE_ORCHESTRATION', False) @@ -224,6 +224,20 @@ def __init__(self): self._environment_variables_dict['ATHENA_ACCOUNT_ACCESS_KEY'] = EnvironmentVariables.get_env('ATHENA_ACCOUNT_ACCESS_KEY', '') self._environment_variables_dict['ATHENA_ACCOUNT_SECRET_KEY'] = EnvironmentVariables.get_env('ATHENA_ACCOUNT_SECRET_KEY', '') + # Cloudability + + self._environment_variables_dict['CLOUDABILITY_VIEW_ID'] = EnvironmentVariables.get_env('CLOUDABILITY_VIEW_ID', '') + self._environment_variables_dict['APPITO_ENVID'] = EnvironmentVariables.get_env('APPITO_ENVID', '') + self._environment_variables_dict['APPITO_KEY_SECRET'] = EnvironmentVariables.get_env('APPITO_KEY_SECRET', '') + self._environment_variables_dict['APPITO_KEY_ACCESS'] = EnvironmentVariables.get_env('APPITO_KEY_ACCESS', '') + self._environment_variables_dict['CLOUDABILITY_API'] = EnvironmentVariables.get_env('CLOUDABILITY_API', '') + self._environment_variables_dict['CLOUDABILITY_API_REPORTS_PATH'] = EnvironmentVariables.get_env('CLOUDABILITY_API_REPORTS_PATH', '') + self._environment_variables_dict['CLOUDABILITY_METRICS'] = EnvironmentVariables.get_env('CLOUDABILITY_METRICS', 'unblended_cost') + self._environment_variables_dict['CLOUDABILITY_DIMENSIONS'] = EnvironmentVariables.get_env('CLOUDABILITY_DIMENSIONS', 'date,category4,vendor_account_name,vendor_account_identifier,vendor') + + + + @staticmethod def to_bool(arg, def_val: bool = None): if isinstance(arg, bool): diff --git a/cloud_governance/policy/common_policies/cloudability_cost_reports.py b/cloud_governance/policy/common_policies/cloudability_cost_reports.py new file mode 100644 index 00000000..e0cedba6 --- /dev/null +++ b/cloud_governance/policy/common_policies/cloudability_cost_reports.py @@ -0,0 +1,211 @@ +import datetime +import json + +from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload +from cloud_governance.common.google_drive.gcp_operations import GCPOperations +from cloud_governance.common.logger.init_logger import logger +from cloud_governance.common.utils.api_requests import APIRequests +from cloud_governance.common.utils.configs import LOOK_BACK_DAYS, PUBLIC_ACCOUNTS_COST_REPORTS, MONTHS, \ + DEFAULT_ROUND_DIGITS, DATE_FORMAT +from cloud_governance.common.utils.utils import Utils +from cloud_governance.main.environment_variables import environment_variables + + +class CloudabilityCostReports: + """ + This class performs cloudability cost operations + """ + + APPITO_LOGIN_API = "https://frontdoor.apptio.com/service/apikeylogin" + + def __init__(self): + self.__api_requests = APIRequests() + self.__environment_variables_dict = environment_variables.environment_variables_dict + self.__view_id = self.__environment_variables_dict.get('CLOUDABILITY_VIEW_ID') + self.__dimensions = self.__environment_variables_dict.get('CLOUDABILITY_DIMENSIONS') + self.__metrics = self.__environment_variables_dict.get('CLOUDABILITY_METRICS') + self.__cloudability_api = self.__environment_variables_dict.get('CLOUDABILITY_API') + self.__reports_path = self.__environment_variables_dict.get('CLOUDABILITY_API_REPORTS_PATH') + self.__appito_envid = self.__environment_variables_dict.get('APPITO_ENVID') + self.__key_secret = self.__environment_variables_dict.get('APPITO_KEY_SECRET') + self.__key_access = self.__environment_variables_dict.get('APPITO_KEY_ACCESS') + self.__gcp_operations = GCPOperations() + self.elastic_upload = ElasticUpload() + + def __get_appito_token(self): + """ + This method returns the appito token + :return: + :rtype: + """ + data = { + "keyAccess": self.__key_access, + "keySecret": self.__key_secret + } + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + } + response = self.__api_requests.post(url=self.APPITO_LOGIN_API, data=json.dumps(data), headers=headers) + if response.ok: + return response.headers['apptio-opentoken'] + return None + + def __get_start_date(self): + return (self.__get_end_date() - datetime.timedelta(days=LOOK_BACK_DAYS)).replace(day=1) + + def __get_end_date(self): + return datetime.datetime.utcnow().date() + + def __get_cost_reports(self, start_date: str = None, end_date: str = None, custom_filter: str = ''): + """ + This method returns the cost reports from the cloudability + :return: + :rtype: + """ + appito_token = self.__get_appito_token() + if not appito_token: + raise Exception("Appito Token missing error") + if not start_date: + start_date = self.__get_start_date() + if not end_date: + end_date = self.__get_end_date() + api = (f'{self.__cloudability_api}/{self.__reports_path}?' + f'dimensions={self.__dimensions}&metrics={self.__metrics}' + f'&start_date={start_date}&end_date={end_date}' + f'&id={self.__view_id}&{custom_filter}') + headers = { + "apptio-environmentid": self.__appito_envid, + "apptio-opentoken": appito_token + } + + response = self.__api_requests.get(url=api, headers=headers) + if isinstance(response, dict): + return response.get('results') + return {} + + def __get_analysed_reports(self): + """ + This method returns the cost reports by adding actual usage, allocated budget + :return: + :rtype: + """ + accounts_reports_df = self.__gcp_operations.get_accounts_sheet(sheet_name=PUBLIC_ACCOUNTS_COST_REPORTS) + cost_centers = list(set(accounts_reports_df['CostCenter'].tolist())) + custom_filter = '&'.join([f'filters=category4=={cost_center}' for cost_center in cost_centers]) + cloudability_reports = self.__get_cost_reports(custom_filter=custom_filter) + cost_reports = {} + for account in cloudability_reports: + account['date'] = (datetime.datetime.strptime(account.get('date', ''), DATE_FORMAT) + .replace(day=1).date().__str__()) + account_id = account.get('vendor_account_identifier').replace('-', '') + account_row = accounts_reports_df[accounts_reports_df['AccountId'] == account_id].reset_index().to_dict( + orient='records') + if account_row: + timestamp = datetime.datetime.strptime(account.get('date'), '%Y-%m-%d') + account_data = account_row[0] + cost_center = account_data.get('CostCenter', 0) + account_budget = round(float(account_data.get('Budget', '0').replace(',', ''))) + year = str(account_data.get('Year')) + account_owner = account_data.get('Owner') + cloud_name = account.get('vendor').upper() + if Utils.equal_ignore_case(cloud_name, 'amazon'): + cloud_name = 'AWS' + month = datetime.datetime.strftime(timestamp, '%Y %b') + index_id = f"""{account.get('date')}-{account.get('vendor_account_name').lower()}""" + + if year in account.get('date'): + if index_id not in cost_reports: + cost_reports.setdefault(index_id, {}).update({ + 'Account': account.get('vendor_account_name'), + 'Actual': round(float(account.get('unblended_cost')), DEFAULT_ROUND_DIGITS), + 'AccountId': account_id, + 'Owner': account_owner, + 'start_date': account.get('date'), + 'CloudName': cloud_name, + 'Forecast': 0, + 'index_id': f"""{account.get('date')}-{account.get('vendor_account_name').lower()}""", + 'timestamp': timestamp, + 'Month': month, + 'Budget': round(account_budget / MONTHS, DEFAULT_ROUND_DIGITS), + 'AllocatedBudget': account_budget, + 'CostCenter': cost_center, + 'filter_date': f'{account.get("date")}-{month.split()[-1]}' + }) + else: + cost_reports[index_id]['Actual'] += round(float(account.get('unblended_cost')), + DEFAULT_ROUND_DIGITS) + return list(cost_reports.values()) + + def __next_twelve_months(self): + """ + This method returns the next 12 months, year + :return: + """ + year = datetime.datetime.utcnow().year + next_month = datetime.datetime.utcnow().month + 1 + month_year = [] + for idx in range(MONTHS): + month = str((idx + next_month) % MONTHS) + c_year = year + if len(month) == 1: + month = f'0{month}' + if month == '00': + month = 12 + year = year+1 + month_year.append((str(month), c_year)) + return month_year + + def __forecast_for_next_months(self, cost_data: list): + """ + This method returns the forecast of next twelve months data + :param cost_data: + :return: + """ + forecast_cost_data = [] + month_years = self.__next_twelve_months() + month = (datetime.datetime.utcnow().month - 1) % 12 + if month == 0: + month = 12 + if len(str(month)) == 1: + month = f'0{month}' + year = datetime.datetime.utcnow().year + cache_start_date = f'{year}-{str(month)}-01' + for data in cost_data: + if cache_start_date == data.get('start_date') and data.get('CostCenter') > 0: + for m_y in month_years: + m, y = m_y[0], m_y[1] + start_date = f'{y}-{m}-01' + timestamp = datetime.datetime.strptime(start_date, "%Y-%m-%d") + index_id = f'{start_date}-{data.get("Account").lower()}' + month = datetime.datetime.strftime(timestamp, "%Y %b") + forecast_cost_data.append({ + **data, + 'Actual': 0, + 'start_date': start_date, + 'timestamp': timestamp, + 'index_id': index_id, + 'filter_date': f'{start_date}-{month.split()[-1]}', + 'Month': month} + ) + return forecast_cost_data + + def __get_cost_and_upload(self): + """ + This method collect the cost and uploads to the ElasticSearch + :return: + """ + collected_data = self.__get_analysed_reports() + forecast_data = self.__forecast_for_next_months(cost_data=collected_data) + upload_data = collected_data + forecast_data + self.elastic_upload.es_upload_data(items=upload_data, set_index='index_id') + return upload_data + + def run(self): + """ + This is the starting of the operations + :return: + :rtype: + """ + logger.info(f'Cloudability Cost Reports=> ReportsPath: {self.__reports_path}, Metrics: {self.__metrics}, API: {self.__cloudability_api}') + self.__get_cost_and_upload() diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile b/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile index 5ccc1b1e..bf52b725 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile +++ b/jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile @@ -23,6 +23,14 @@ pipeline { ATHENA_DATABASE_NAME = credentials('ATHENA_DATABASE_NAME') ATHENA_TABLE_NAME = credentials('ATHENA_TABLE_NAME') + CLOUDABILITY_API = credentials('cloudability_api') + CLOUDABILITY_API_REPORTS_PATH = credentials('cloudability_api_reports_path') + CLOUDABILITY_METRICS = credentials('cloudability_metrics') + CLOUDABILITY_VIEW_ID = credentials('cloudability_view_id') + APPITO_KEY_ACCESS = credentials('appito_key_access') + APPITO_KEY_SECRET = credentials('appito_key_secret') + APPITO_ENVID = credentials('appito_envid') + contact1 = "ebattat@redhat.com" contact2 = "athiruma@redhat.com" } diff --git a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py index f9660ccc..23eeb1a9 100644 --- a/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py +++ b/jenkins/clouds/aws/daily/org_cost_explorer/run_org_upload_es.py @@ -1,7 +1,5 @@ - import os - AWS_ACCESS_KEY_ID_DELETE_PERF = os.environ['AWS_ACCESS_KEY_ID_DELETE_PERF'] AWS_SECRET_ACCESS_KEY_DELETE_PERF = os.environ['AWS_SECRET_ACCESS_KEY_DELETE_PERF'] ES_HOST = os.environ['ES_HOST'] @@ -18,20 +16,33 @@ ATHENA_DATABASE_NAME = os.environ['ATHENA_DATABASE_NAME'] ATHENA_TABLE_NAME = os.environ['ATHENA_TABLE_NAME'] +# Cloudability env variables + +CLOUDABILITY_API = os.environ['CLOUDABILITY_API'] +CLOUDABILITY_API_REPORTS_PATH = os.environ['CLOUDABILITY_API_REPORTS_PATH'] +CLOUDABILITY_METRICS = os.environ['CLOUDABILITY_METRICS'] +CLOUDABILITY_VIEW_ID = os.environ['CLOUDABILITY_VIEW_ID'] +APPITO_KEY_ACCESS = os.environ['APPITO_KEY_ACCESS'] +APPITO_KEY_SECRET = os.environ['APPITO_KEY_SECRET'] +APPITO_ENVID = os.environ['APPITO_ENVID'] + + os.system('echo "Updating the Org level cost billing reports"') # Cost Explorer upload to ElasticSearch cost_metric = 'UnblendedCost' # UnblendedCost/BlendedCost granularity = 'DAILY' # DAILY/MONTHLY/HOURLY - -common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-global-cost-billing-reports', 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, 'COST_CENTER_OWNER': f"{COST_CENTER_OWNER}", 'REPLACE_ACCOUNT_NAME': REPLACE_ACCOUNT_NAME, 'PAYER_SUPPORT_FEE_CREDIT': PAYER_SUPPORT_FEE_CREDIT} +common_input_vars = {'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': 'cloud-governance-global-cost-billing-reports', + 'log_level': 'INFO', 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, + 'COST_CENTER_OWNER': f"{COST_CENTER_OWNER}", 'REPLACE_ACCOUNT_NAME': REPLACE_ACCOUNT_NAME, + 'PAYER_SUPPORT_FEE_CREDIT': PAYER_SUPPORT_FEE_CREDIT} combine_vars = lambda item: f'{item[0]}="{item[1]}"' common_input_vars['es_index'] = 'cloud-governance-clouds-billing-reports' common_envs = list(map(combine_vars, common_input_vars.items())) -os.system(f"""podman run --rm --name cloud-governance -e policy="cost_explorer_payer_billings" -e AWS_ACCOUNT_ROLE="{AWS_ACCOUNT_ROLE}" -e account="PERF-DEPT" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") - +os.system( + f"""podman run --rm --name cloud-governance -e policy="cost_explorer_payer_billings" -e AWS_ACCOUNT_ROLE="{AWS_ACCOUNT_ROLE}" -e account="PERF-DEPT" -e AWS_ACCESS_KEY_ID="{AWS_ACCESS_KEY_ID_DELETE_PERF}" -e AWS_SECRET_ACCESS_KEY="{AWS_SECRET_ACCESS_KEY_DELETE_PERF}" -e SPREADSHEET_ID="{COST_SPREADSHEET_ID}" -e {' -e '.join(common_envs)} -v "{GOOGLE_APPLICATION_CREDENTIALS}":"{GOOGLE_APPLICATION_CREDENTIALS}" quay.io/ebattat/cloud-governance:latest""") os.system('echo "Run the Spot Analysis report over the account using AWS Athena"') os.system(f"""podman run --rm --name cloud-governance -e policy="spot_savings_analysis" -e account="pnt-payer" \ @@ -43,3 +54,63 @@ -e ATHENA_DATABASE_NAME="{ATHENA_DATABASE_NAME}" \ -e ATHENA_TABLE_NAME="{ATHENA_TABLE_NAME}" \ quay.io/ebattat/cloud-governance:latest""") + +CLOUD_GOVERNANCE_IMAGE = "quay.io/ebattat/cloud-governance:latest" +CONTAINER_NAME = "cloud-governance" +COST_ES_INDEX = "cloud-governance-clouds-billing-reports" +CLOUDABILITY_POLICY = 'cloudability_cost_reports' + + +def run_shell_cmd(cmd: str): + """ + This method run the shell command + :param cmd: + :type cmd: + :return: + :rtype: + """ + os.system(cmd) + + +def generate_shell_cmd(policy: str, env_variables: dict, mounted_volumes: str = ''): + """ + This method returns the shell command + :param mounted_volumes: + :type mounted_volumes: + :param env_variables: + :type env_variables: + :param policy: + :type policy: + :return: + :rtype: + """ + inject_container_envs = ' '.join(list(map(lambda item: f'-e {item[0]}="{item[1]}"', env_variables.items()))) + return (f'podman run --rm --name {CONTAINER_NAME} -e policy="{policy}" {inject_container_envs} {mounted_volumes} ' + f'{CLOUD_GOVERNANCE_IMAGE}') + + +common_env_vars = { + 'es_host': ES_HOST, 'es_port': ES_PORT, 'es_index': COST_ES_INDEX, + 'GOOGLE_APPLICATION_CREDENTIALS': GOOGLE_APPLICATION_CREDENTIALS, + 'SPREADSHEET_ID': COST_SPREADSHEET_ID, +} + +cloudability_env_vars = { + 'CLOUDABILITY_API': CLOUDABILITY_API, + 'CLOUDABILITY_API_REPORTS_PATH': CLOUDABILITY_API_REPORTS_PATH, + 'CLOUDABILITY_METRICS': CLOUDABILITY_METRICS, + 'CLOUDABILITY_VIEW_ID': CLOUDABILITY_VIEW_ID, + 'APPITO_KEY_ACCESS': APPITO_KEY_ACCESS, + 'APPITO_KEY_SECRET': APPITO_KEY_SECRET, + 'APPITO_ENVID': APPITO_ENVID, +} + +mounted_volumes = f" -v {GOOGLE_APPLICATION_CREDENTIALS}:{GOOGLE_APPLICATION_CREDENTIALS}" +cloudability_run_command = generate_shell_cmd(policy=CLOUDABILITY_POLICY, + env_variables={ + **common_env_vars, + **cloudability_env_vars + }, mounted_volumes=mounted_volumes) + +run_shell_cmd(f"echo Running the {CLOUDABILITY_POLICY}") +run_shell_cmd(cloudability_run_command)