Added the cloudability cost reports (#742)

cloud_governance/common/google_drive/gcp_operations.py

@@ -0,0 +1,26 @@
+import os
+import tempfile
+
+import pandas as pd
+
+from cloud_governance.common.google_drive.google_drive_operations import GoogleDriveOperations
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class GCPOperations:
+
+    def __init__(self):
+        self.__environment_variables_dict = environment_variables.environment_variables_dict
+        self.__gsheet_operations = GoogleDriveOperations()
+        self.__gsheet_id = self.__environment_variables_dict.get('SPREADSHEET_ID', '')
+
+    def get_accounts_sheet(self, sheet_name: str, dir_path: str = None):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dir_path = dir_path if dir_path else tmp_dir
+            file_path = f'{dir_path}/{sheet_name}.csv'
+            if not os.path.exists(file_path):
+                self.__gsheet_operations.download_spreadsheet(spreadsheet_id=self.__gsheet_id,
+                                                              sheet_name=sheet_name,
+                                                              file_path=dir_path)
+            accounts_df = pd.read_csv(file_path)
+            return accounts_df

cloud_governance/common/utils/api_requests.py

@@ -0,0 +1,29 @@
+import asyncio
+
+import aiohttp
+import requests
+
+from cloud_governance.common.logger.init_logger import logger
+
+
+class APIRequests:
+
+    def __init__(self):
+        self.__loop = asyncio.new_event_loop()
+
+    def get(self, url: str, **kwargs):
+        try:
+            response = requests.get(url, **kwargs)
+            if response.ok:
+                return response.json()
+            else:
+                return response.text
+        except Exception as err:
+            raise err
+
+    def post(self, url: str,  **kwargs):
+        try:
+            response = requests.post(url, **kwargs)
+            return response
+        except Exception as err:
+            raise err

cloud_governance/common/utils/configs.py

@@ -1,3 +1,9 @@
 
 
 LOOK_BACK_DAYS = 30
+MONTHS = 12
+DEFAULT_ROUND_DIGITS = 3
+PUBLIC_ACCOUNTS_COST_REPORTS = 'Accounts'
+
+
+DATE_FORMAT = "%Y-%m-%d"

cloud_governance/main/environment_variables.py

@@ -198,7 +198,7 @@ def __init__(self):
         self._environment_variables_dict['KERBEROS_USERS'] = literal_eval(EnvironmentVariables.get_env('KERBEROS_USERS', '[]'))
         self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval(EnvironmentVariables.get_env('POLICIES_TO_ALERT', '[]'))
         self._environment_variables_dict['ADMIN_MAIL_LIST'] = EnvironmentVariables.get_env('ADMIN_MAIL_LIST', '')
-        if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts']:
+        if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts', 'cloudability_cost_reports']:
             self._environment_variables_dict['COMMON_POLICIES'] = True
         # CRO -- Cloud Resource Orch
         self._environment_variables_dict['CLOUD_RESOURCE_ORCHESTRATION'] = EnvironmentVariables.get_boolean_from_environment('CLOUD_RESOURCE_ORCHESTRATION', False)
@@ -226,6 +226,20 @@ def __init__(self):
         self._environment_variables_dict['ATHENA_ACCOUNT_ACCESS_KEY'] = EnvironmentVariables.get_env('ATHENA_ACCOUNT_ACCESS_KEY', '')
         self._environment_variables_dict['ATHENA_ACCOUNT_SECRET_KEY'] = EnvironmentVariables.get_env('ATHENA_ACCOUNT_SECRET_KEY', '')
 
+        # Cloudability
+
+        self._environment_variables_dict['CLOUDABILITY_VIEW_ID'] = EnvironmentVariables.get_env('CLOUDABILITY_VIEW_ID', '')
+        self._environment_variables_dict['APPITO_ENVID'] = EnvironmentVariables.get_env('APPITO_ENVID', '')
+        self._environment_variables_dict['APPITO_KEY_SECRET'] = EnvironmentVariables.get_env('APPITO_KEY_SECRET', '')
+        self._environment_variables_dict['APPITO_KEY_ACCESS'] = EnvironmentVariables.get_env('APPITO_KEY_ACCESS', '')
+        self._environment_variables_dict['CLOUDABILITY_API'] = EnvironmentVariables.get_env('CLOUDABILITY_API', '')
+        self._environment_variables_dict['CLOUDABILITY_API_REPORTS_PATH'] = EnvironmentVariables.get_env('CLOUDABILITY_API_REPORTS_PATH', '')
+        self._environment_variables_dict['CLOUDABILITY_METRICS'] = EnvironmentVariables.get_env('CLOUDABILITY_METRICS', 'unblended_cost')
+        self._environment_variables_dict['CLOUDABILITY_DIMENSIONS'] = EnvironmentVariables.get_env('CLOUDABILITY_DIMENSIONS', 'date,category4,vendor_account_name,vendor_account_identifier,vendor')
+
+
+
+
     @staticmethod
     def to_bool(arg, def_val: bool = None):
         if isinstance(arg, bool):

cloud_governance/policy/common_policies/cloudability_cost_reports.py

@@ -0,0 +1,211 @@
+import datetime
+import json
+
+from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
+from cloud_governance.common.google_drive.gcp_operations import GCPOperations
+from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.common.utils.api_requests import APIRequests
+from cloud_governance.common.utils.configs import LOOK_BACK_DAYS, PUBLIC_ACCOUNTS_COST_REPORTS, MONTHS, \
+    DEFAULT_ROUND_DIGITS, DATE_FORMAT
+from cloud_governance.common.utils.utils import Utils
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class CloudabilityCostReports:
+    """
+    This class performs cloudability cost operations
+    """
+
+    APPITO_LOGIN_API = "https://frontdoor.apptio.com/service/apikeylogin"
+
+    def __init__(self):
+        self.__api_requests = APIRequests()
+        self.__environment_variables_dict = environment_variables.environment_variables_dict
+        self.__view_id = self.__environment_variables_dict.get('CLOUDABILITY_VIEW_ID')
+        self.__dimensions = self.__environment_variables_dict.get('CLOUDABILITY_DIMENSIONS')
+        self.__metrics = self.__environment_variables_dict.get('CLOUDABILITY_METRICS')
+        self.__cloudability_api = self.__environment_variables_dict.get('CLOUDABILITY_API')
+        self.__reports_path = self.__environment_variables_dict.get('CLOUDABILITY_API_REPORTS_PATH')
+        self.__appito_envid = self.__environment_variables_dict.get('APPITO_ENVID')
+        self.__key_secret = self.__environment_variables_dict.get('APPITO_KEY_SECRET')
+        self.__key_access = self.__environment_variables_dict.get('APPITO_KEY_ACCESS')
+        self.__gcp_operations = GCPOperations()
+        self.elastic_upload = ElasticUpload()
+
+    def __get_appito_token(self):
+        """
+        This method returns the appito token
+        :return:
+        :rtype:
+        """
+        data = {
+            "keyAccess": self.__key_access,
+            "keySecret": self.__key_secret
+        }
+        headers = {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json'
+        }
+        response = self.__api_requests.post(url=self.APPITO_LOGIN_API, data=json.dumps(data), headers=headers)
+        if response.ok:
+            return response.headers['apptio-opentoken']
+        return None
+
+    def __get_start_date(self):
+        return (self.__get_end_date() - datetime.timedelta(days=LOOK_BACK_DAYS)).replace(day=1)
+
+    def __get_end_date(self):
+        return datetime.datetime.utcnow().date()
+
+    def __get_cost_reports(self, start_date: str = None, end_date: str = None, custom_filter: str = ''):
+        """
+        This method returns the cost reports from the cloudability
+        :return:
+        :rtype:
+        """
+        appito_token = self.__get_appito_token()
+        if not appito_token:
+            raise Exception("Appito Token missing error")
+        if not start_date:
+            start_date = self.__get_start_date()
+        if not end_date:
+            end_date = self.__get_end_date()
+        api = (f'{self.__cloudability_api}/{self.__reports_path}?'
+               f'dimensions={self.__dimensions}&metrics={self.__metrics}'
+               f'&start_date={start_date}&end_date={end_date}'
+               f'&id={self.__view_id}&{custom_filter}')
+        headers = {
+            "apptio-environmentid": self.__appito_envid,
+            "apptio-opentoken": appito_token
+        }
+
+        response = self.__api_requests.get(url=api, headers=headers)
+        if isinstance(response, dict):
+            return response.get('results')
+        return {}
+
+    def __get_analysed_reports(self):
+        """
+        This method returns the cost reports by adding actual usage, allocated budget
+        :return:
+        :rtype:
+        """
+        accounts_reports_df = self.__gcp_operations.get_accounts_sheet(sheet_name=PUBLIC_ACCOUNTS_COST_REPORTS)
+        cost_centers = list(set(accounts_reports_df['CostCenter'].tolist()))
+        custom_filter = '&'.join([f'filters=category4=={cost_center}' for cost_center in cost_centers])
+        cloudability_reports = self.__get_cost_reports(custom_filter=custom_filter)
+        cost_reports = {}
+        for account in cloudability_reports:
+            account['date'] = (datetime.datetime.strptime(account.get('date', ''), DATE_FORMAT)
+                               .replace(day=1).date().__str__())
+            account_id = account.get('vendor_account_identifier').replace('-', '')
+            account_row = accounts_reports_df[accounts_reports_df['AccountId'] == account_id].reset_index().to_dict(
+                orient='records')
+            if account_row:
+                timestamp = datetime.datetime.strptime(account.get('date'), '%Y-%m-%d')
+                account_data = account_row[0]
+                cost_center = account_data.get('CostCenter', 0)
+                account_budget = round(float(account_data.get('Budget', '0').replace(',', '')))
+                year = str(account_data.get('Year'))
+                account_owner = account_data.get('Owner')
+                cloud_name = account.get('vendor').upper()
+                if Utils.equal_ignore_case(cloud_name, 'amazon'):
+                    cloud_name = 'AWS'
+                month = datetime.datetime.strftime(timestamp, '%Y %b')
+                index_id = f"""{account.get('date')}-{account.get('vendor_account_name').lower()}"""
+
+                if year in account.get('date'):
+                    if index_id not in cost_reports:
+                        cost_reports.setdefault(index_id, {}).update({
+                            'Account': account.get('vendor_account_name'),
+                            'Actual': round(float(account.get('unblended_cost')), DEFAULT_ROUND_DIGITS),
+                            'AccountId': account_id,
+                            'Owner': account_owner,
+                            'start_date': account.get('date'),
+                            'CloudName': cloud_name,
+                            'Forecast': 0,
+                            'index_id': f"""{account.get('date')}-{account.get('vendor_account_name').lower()}""",
+                            'timestamp': timestamp,
+                            'Month': month,
+                            'Budget': round(account_budget / MONTHS, DEFAULT_ROUND_DIGITS),
+                            'AllocatedBudget': account_budget,
+                            'CostCenter': cost_center,
+                            'filter_date': f'{account.get("date")}-{month.split()[-1]}'
+                        })
+                    else:
+                        cost_reports[index_id]['Actual'] += round(float(account.get('unblended_cost')),
+                                                                  DEFAULT_ROUND_DIGITS)
+        return list(cost_reports.values())
+
+    def __next_twelve_months(self):
+        """
+        This method returns the next 12 months, year
+        :return:
+        """
+        year = datetime.datetime.utcnow().year
+        next_month = datetime.datetime.utcnow().month + 1
+        month_year = []
+        for idx in range(MONTHS):
+            month = str((idx + next_month) % MONTHS)
+            c_year = year
+            if len(month) == 1:
+                month = f'0{month}'
+            if month == '00':
+                month = 12
+                year = year+1
+            month_year.append((str(month), c_year))
+        return month_year
+
+    def __forecast_for_next_months(self, cost_data: list):
+        """
+        This method returns the forecast of next twelve months data
+        :param cost_data:
+        :return:
+        """
+        forecast_cost_data = []
+        month_years = self.__next_twelve_months()
+        month = (datetime.datetime.utcnow().month - 1) % 12
+        if month == 0:
+            month = 12
+        if len(str(month)) == 1:
+            month = f'0{month}'
+        year = datetime.datetime.utcnow().year
+        cache_start_date = f'{year}-{str(month)}-01'
+        for data in cost_data:
+            if cache_start_date == data.get('start_date') and data.get('CostCenter') > 0:
+                for m_y in month_years:
+                    m, y = m_y[0], m_y[1]
+                    start_date = f'{y}-{m}-01'
+                    timestamp = datetime.datetime.strptime(start_date, "%Y-%m-%d")
+                    index_id = f'{start_date}-{data.get("Account").lower()}'
+                    month = datetime.datetime.strftime(timestamp, "%Y %b")
+                    forecast_cost_data.append({
+                        **data,
+                        'Actual': 0,
+                        'start_date': start_date,
+                        'timestamp': timestamp,
+                        'index_id': index_id,
+                        'filter_date': f'{start_date}-{month.split()[-1]}',
+                        'Month': month}
+                    )
+        return forecast_cost_data
+
+    def __get_cost_and_upload(self):
+        """
+        This method collect the cost and uploads to the ElasticSearch
+        :return:
+        """
+        collected_data = self.__get_analysed_reports()
+        forecast_data = self.__forecast_for_next_months(cost_data=collected_data)
+        upload_data = collected_data + forecast_data
+        self.elastic_upload.es_upload_data(items=upload_data, set_index='index_id')
+        return upload_data
+
+    def run(self):
+        """
+        This is the starting of the operations
+        :return:
+        :rtype:
+        """
+        logger.info(f'Cloudability Cost Reports=> ReportsPath: {self.__reports_path}, Metrics: {self.__metrics}, API: {self.__cloudability_api}')
+        self.__get_cost_and_upload()

jenkins/clouds/aws/daily/org_cost_explorer/Jenkinsfile

@@ -23,6 +23,14 @@ pipeline {
         ATHENA_DATABASE_NAME = credentials('ATHENA_DATABASE_NAME')
         ATHENA_TABLE_NAME = credentials('ATHENA_TABLE_NAME')
 
+        CLOUDABILITY_API = credentials('cloudability_api')
+        CLOUDABILITY_API_REPORTS_PATH = credentials('cloudability_api_reports_path')
+        CLOUDABILITY_METRICS = credentials('cloudability_metrics')
+        CLOUDABILITY_VIEW_ID = credentials('cloudability_view_id')
+        APPITO_KEY_ACCESS = credentials('appito_key_access')
+        APPITO_KEY_SECRET = credentials('appito_key_secret')
+        APPITO_ENVID = credentials('appito_envid')
+
         contact1 = "[email protected]"
         contact2 = "[email protected]"
     }