Check the previous cost before sending an alert (#883)

cloud_governance/cloud_resource_orchestration/common/abstract_monitor_tickets.py

@@ -192,13 +192,13 @@ def _monitor_ticket_duration(self, ticket_id: str, region_name: str, duration: i
             self.__postfix.send_email_postfix(to=user, cc=cc, subject=subject, content=body, mime_type='html',
                                               message_type=message_type)
 
-    def get_budget_exceed_alert_times(self, user: str):
+    def get_budget_exceed_alert_times(self, user: str, remaining_budget: int):
         """
-        This method returns the number of times alerts send to an user
+        This method returns the number of times alerts send to a user
         :return:
         """
         current_date = datetime.now(timezone.utc).date()
-        start_date = current_date - timedelta(days=10)
+        start_date = current_date - timedelta(days=15)
         query = {
             "query": {
                 "bool": {
@@ -224,7 +224,10 @@ def get_budget_exceed_alert_times(self, user: str):
                                                                es_index=self.CLOUD_GOVERNANCE_ES_MAIL_INDEX,
                                                                search_size=10,
                                                                limit_to_size=True)
-        return len(response)
+        if response:
+            if 'remaining_budget' in response[0]['_source']:
+                return remaining_budget < int(response[0]['_source'].get('remaining_budget')), len(response)
+        return True, len(response)
 
     @typeguard.typechecked
     @logger_time_stamp
@@ -241,16 +244,16 @@ def _monitor_ticket_budget(self, ticket_id: str, region_name: str, budget: int,
         remaining_budget = budget - used_budget
         threshold_budget = budget - (budget * (self.__ticket_over_usage_limit / 100))
         subject = body = None
-        alerted_times = self.get_budget_exceed_alert_times(user=user)
-        if threshold_budget >= remaining_budget > 0 and alerted_times < 3:
+        alert_user, total_alerts = self.get_budget_exceed_alert_times(user=user, remaining_budget=remaining_budget)
+        if threshold_budget >= remaining_budget > 0 and alert_user and total_alerts < 2:
             ticket_extended = self.extend_tickets_budget(ticket_id=ticket_id, region_name=region_name,
                                                          current_budget=budget)
             if not ticket_extended:
                 subject, body = self.__mail_message.cro_monitor_budget_remain_alert(user=user, budget=budget,
                                                                                     ticket_id=ticket_id,
                                                                                     used_budget=used_budget,
                                                                                     remain_budget=remaining_budget)
-        elif remaining_budget <= 0 and alerted_times == 2:
+        elif remaining_budget <= 0 and alert_user and total_alerts == 1:
             ticket_extended = self.extend_tickets_budget(ticket_id=ticket_id, region_name=region_name,
                                                          current_budget=budget)
             if not ticket_extended:
@@ -260,7 +263,7 @@ def _monitor_ticket_budget(self, ticket_id: str, region_name: str, budget: int,
                                                                                          remain_budget=remaining_budget)
         if subject and body:
             self.__postfix.send_email_postfix(to=user, cc=cc, subject=subject, content=body, mime_type='html',
-                                              message_type='budget_exceed_alert')
+                                              message_type='budget_exceed_alert', remaining_budget=remaining_budget)
 
     @logger_time_stamp
     def _monitor_in_progress_tickets(self):

cloud_governance/common/mails/postfix.py

@@ -145,18 +145,22 @@ def send_email_postfix(self, subject: str, to: any, cc: list, content: str, **kw
                             date_key = datetime.datetime.now().strftime("%Y%m%d%H")
                             if self.__policy_output:
                                 self.__s3_operations.upload_file(file_name_path=kwargs['filename'],
-                                                                 bucket=self.bucket_name, key=f'{self.key}/{self.__policy}/{date_key}',
+                                                                 bucket=self.bucket_name,
+                                                                 key=f'{self.key}/{self.__policy}/{date_key}',
                                                                  upload_file=file_name)
                                 s3_path = f'{self.__policy_output}/logs/{self.__policy}/{date_key}/{file_name}'
                                 content += f'\n\nresource_file_path: s3://{s3_path}\n\n'
                         es_data = kwargs.get('es_data')
-                        data = {'Policy': self.__policy, 'To': to, 'Cc': cc, 'Message': content, 'Account': self.__account.upper(), 'MessageType': kwargs.get('message_type', 'alert')}
+                        data = {'Policy': self.__policy, 'To': to, 'Cc': cc, 'Message': content,
+                                'Account': self.__account.upper(), 'MessageType': kwargs.get('message_type', 'alert')}
                         if es_data:
                             data.update(es_data)
                         if kwargs.get('resource_id'):
                             data['resource_id'] = kwargs['resource_id']
                         if kwargs.get('extra_purse'):
                             data['extra_purse'] = round(kwargs['extra_purse'], 3)
+                        if kwargs.get('remaining_budget'):
+                            data['remaining_budget'] = kwargs['remaining_budget']
                         if self.__es_host:
                             self.__es_operations.upload_to_elasticsearch(data=data, index=self.__es_index)
                             logger.warn(f'Uploaded to es index: {self.__es_index}')