From 5d62cbec1ffeebb4acd35ae587fb42fecd9e0d65 Mon Sep 17 00:00:00 2001 From: Thirumalesh Aaraveti <athiruma@redhat.com> Date: Mon, 9 Dec 2024 17:35:23 +0530 Subject: [PATCH] Check the previous cost before sending an alert --- .../common/abstract_monitor_tickets.py | 19 +++++++++++-------- cloud_governance/common/mails/postfix.py | 8 ++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cloud_governance/cloud_resource_orchestration/common/abstract_monitor_tickets.py b/cloud_governance/cloud_resource_orchestration/common/abstract_monitor_tickets.py index 9d9ded26..a0a8ea30 100644 --- a/cloud_governance/cloud_resource_orchestration/common/abstract_monitor_tickets.py +++ b/cloud_governance/cloud_resource_orchestration/common/abstract_monitor_tickets.py @@ -192,13 +192,13 @@ def _monitor_ticket_duration(self, ticket_id: str, region_name: str, duration: i self.__postfix.send_email_postfix(to=user, cc=cc, subject=subject, content=body, mime_type='html', message_type=message_type) - def get_budget_exceed_alert_times(self, user: str): + def get_budget_exceed_alert_times(self, user: str, remaining_budget: int): """ - This method returns the number of times alerts send to an user + This method returns the number of times alerts send to a user :return: """ current_date = datetime.now(timezone.utc).date() - start_date = current_date - timedelta(days=10) + start_date = current_date - timedelta(days=15) query = { "query": { "bool": { @@ -224,7 +224,10 @@ def get_budget_exceed_alert_times(self, user: str): es_index=self.CLOUD_GOVERNANCE_ES_MAIL_INDEX, search_size=10, limit_to_size=True) - return len(response) + if response: + if 'remaining_budget' in response[0]['_source']: + return remaining_budget < int(response[0]['_source'].get('remaining_budget')), len(response) + return True, len(response) @typeguard.typechecked @logger_time_stamp @@ -241,8 +244,8 @@ def _monitor_ticket_budget(self, ticket_id: str, region_name: str, budget: int, remaining_budget = budget - used_budget threshold_budget = budget - (budget * (self.__ticket_over_usage_limit / 100)) subject = body = None - alerted_times = self.get_budget_exceed_alert_times(user=user) - if threshold_budget >= remaining_budget > 0 and alerted_times < 3: + alert_user, total_alerts = self.get_budget_exceed_alert_times(user=user, remaining_budget=remaining_budget) + if threshold_budget >= remaining_budget > 0 and alert_user and total_alerts < 2: ticket_extended = self.extend_tickets_budget(ticket_id=ticket_id, region_name=region_name, current_budget=budget) if not ticket_extended: @@ -250,7 +253,7 @@ def _monitor_ticket_budget(self, ticket_id: str, region_name: str, budget: int, ticket_id=ticket_id, used_budget=used_budget, remain_budget=remaining_budget) - elif remaining_budget <= 0 and alerted_times == 2: + elif remaining_budget <= 0 and alert_user and total_alerts == 1: ticket_extended = self.extend_tickets_budget(ticket_id=ticket_id, region_name=region_name, current_budget=budget) if not ticket_extended: @@ -260,7 +263,7 @@ def _monitor_ticket_budget(self, ticket_id: str, region_name: str, budget: int, remain_budget=remaining_budget) if subject and body: self.__postfix.send_email_postfix(to=user, cc=cc, subject=subject, content=body, mime_type='html', - message_type='budget_exceed_alert') + message_type='budget_exceed_alert', remaining_budget=remaining_budget) @logger_time_stamp def _monitor_in_progress_tickets(self): diff --git a/cloud_governance/common/mails/postfix.py b/cloud_governance/common/mails/postfix.py index d1b001f1..82fb2b4b 100644 --- a/cloud_governance/common/mails/postfix.py +++ b/cloud_governance/common/mails/postfix.py @@ -145,18 +145,22 @@ def send_email_postfix(self, subject: str, to: any, cc: list, content: str, **kw date_key = datetime.datetime.now().strftime("%Y%m%d%H") if self.__policy_output: self.__s3_operations.upload_file(file_name_path=kwargs['filename'], - bucket=self.bucket_name, key=f'{self.key}/{self.__policy}/{date_key}', + bucket=self.bucket_name, + key=f'{self.key}/{self.__policy}/{date_key}', upload_file=file_name) s3_path = f'{self.__policy_output}/logs/{self.__policy}/{date_key}/{file_name}' content += f'\n\nresource_file_path: s3://{s3_path}\n\n' es_data = kwargs.get('es_data') - data = {'Policy': self.__policy, 'To': to, 'Cc': cc, 'Message': content, 'Account': self.__account.upper(), 'MessageType': kwargs.get('message_type', 'alert')} + data = {'Policy': self.__policy, 'To': to, 'Cc': cc, 'Message': content, + 'Account': self.__account.upper(), 'MessageType': kwargs.get('message_type', 'alert')} if es_data: data.update(es_data) if kwargs.get('resource_id'): data['resource_id'] = kwargs['resource_id'] if kwargs.get('extra_purse'): data['extra_purse'] = round(kwargs['extra_purse'], 3) + if kwargs.get('remaining_budget'): + data['remaining_budget'] = kwargs['remaining_budget'] if self.__es_host: self.__es_operations.upload_to_elasticsearch(data=data, index=self.__es_index) logger.warn(f'Uploaded to es index: {self.__es_index}')