From 01f2a70cb94b5846c1933ae3f1635cf2367c90c8 Mon Sep 17 00:00:00 2001 From: Yu Shigetani Date: Fri, 12 Jul 2024 19:37:04 +0900 Subject: [PATCH 01/14] Issue 76 Support Terraform --- terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf b/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf index 41a28a4..a943312 100644 --- a/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf +++ b/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf @@ -673,6 +673,10 @@ resource "aws_lambda_function" "AHA-LambdaFunction-PrimaryRegion" { environment { variables = { + "Slack" = var.SlackWebhookURL != "" ? "True" : null + "Team" = var.MicrosoftTeamsWebhookURL != "" ? "True" : null + "Chime" = var.AmazonChimeWebhookURL != "" ? "True" : null + "Eventbridge" = var.EventBusName != "" ? "True" : null "DYNAMODB_TABLE" = "${var.dynamodbtable}-${random_string.resource_code.result}" "EMAIL_SUBJECT" = var.Subject "EVENT_SEARCH_BACK" = var.EventSearchBack @@ -720,6 +724,10 @@ resource "aws_lambda_function" "AHA-LambdaFunction-SecondaryRegion" { environment { variables = { + "Slack" = var.SlackWebhookURL != "" ? "True" : null + "Team" = var.MicrosoftTeamsWebhookURL != "" ? "True" : null + "Chime" = var.AmazonChimeWebhookURL != "" ? "True" : null + "Eventbridge" = var.EventBusName != "" ? "True" : null "DYNAMODB_TABLE" = "${var.dynamodbtable}-${random_string.resource_code.result}" "EMAIL_SUBJECT" = var.Subject "EVENT_SEARCH_BACK" = var.EventSearchBack From cb97f2df2c096388b74d5a1c32b838d6a6bdc5bb Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 27 Aug 2024 10:29:39 -0400 Subject: [PATCH 02/14] Fix for template file / env vars / eventbridge schedule --- .../Terraform_DEPLOY_AHA.tf | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf b/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf index 41a28a4..b54a0e9 100644 --- a/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf +++ b/terraform/Terraform_DEPLOY_AHA/Terraform_DEPLOY_AHA.tf @@ -29,20 +29,16 @@ provider "aws" { locals { source_files = ["${path.module}/../../handler.py", "${path.module}/../../messagegenerator.py"] } -data "template_file" "t_file" { - count = "${length(local.source_files)}" - template = "${file(element(local.source_files, count.index))}" -} data "archive_file" "lambda_zip" { type = "zip" output_path = "${path.module}/lambda_function.zip" source { filename = "${basename(local.source_files[0])}" - content = "${data.template_file.t_file.0.rendered}" + content = file("${local.source_files[0]}") } source { filename = "${basename(local.source_files[1])}" - content = "${data.template_file.t_file.1.rendered}" + content = file("${local.source_files[1]}") } } @@ -673,6 +669,10 @@ resource "aws_lambda_function" "AHA-LambdaFunction-PrimaryRegion" { environment { variables = { + "Slack" = var.SlackWebhookURL != "" ? "True" : null + "Team" = var.MicrosoftTeamsWebhookURL != "" ? "True" : null + "Chime" = var.AmazonChimeWebhookURL != "" ? "True" : null + "Eventbridge" = var.EventBusName != "" ? "True" : null "DYNAMODB_TABLE" = "${var.dynamodbtable}-${random_string.resource_code.result}" "EMAIL_SUBJECT" = var.Subject "EVENT_SEARCH_BACK" = var.EventSearchBack @@ -681,7 +681,7 @@ resource "aws_lambda_function" "AHA-LambdaFunction-PrimaryRegion" { "ORG_STATUS" = var.AWSOrganizationsEnabled "REGIONS" = var.Regions "TO_EMAIL" = var.ToEmail - "MANAGEMENT_ROLE_ARN" = var.ManagementAccountRoleArn + "MANAGEMENT_ROLE_ARN" = var.ManagementAccountRoleArn == "" ? "None" : var.ManagementAccountRoleArn "ACCOUNT_IDS" = var.ExcludeAccountIDs "S3_BUCKET" = join("",aws_s3_bucket.AHA-S3Bucket-PrimaryRegion[*].bucket) } @@ -720,6 +720,10 @@ resource "aws_lambda_function" "AHA-LambdaFunction-SecondaryRegion" { environment { variables = { + "Slack" = var.SlackWebhookURL != "" ? "True" : null + "Team" = var.MicrosoftTeamsWebhookURL != "" ? "True" : null + "Chime" = var.AmazonChimeWebhookURL != "" ? "True" : null + "Eventbridge" = var.EventBusName != "" ? "True" : null "DYNAMODB_TABLE" = "${var.dynamodbtable}-${random_string.resource_code.result}" "EMAIL_SUBJECT" = var.Subject "EVENT_SEARCH_BACK" = var.EventSearchBack @@ -752,7 +756,7 @@ resource "aws_lambda_function" "AHA-LambdaFunction-SecondaryRegion" { resource "aws_cloudwatch_event_rule" "AHA-LambdaSchedule-PrimaryRegion" { description = "Lambda trigger Event" event_bus_name = "default" - is_enabled = true + state = "ENABLED" name = "AHA-LambdaSchedule-${random_string.resource_code.result}" schedule_expression = "rate(1 minute)" tags = { @@ -760,11 +764,11 @@ resource "aws_cloudwatch_event_rule" "AHA-LambdaSchedule-PrimaryRegion" { } } resource "aws_cloudwatch_event_rule" "AHA-LambdaSchedule-SecondaryRegion" { + description = "Lambda trigger Event" count = "${var.aha_secondary_region == "" ? 0 : 1}" provider = aws.secondary_region - description = "Lambda trigger Event" event_bus_name = "default" - is_enabled = true + state = "ENABLED" name = "AHA-LambdaSchedule-${random_string.resource_code.result}" schedule_expression = "rate(1 minute)" tags = { From 5fbfd945d5e9bdf25c4b2e8cb82f40e294ae86a0 Mon Sep 17 00:00:00 2001 From: Andrew Riley Date: Wed, 18 Sep 2024 11:45:38 -0400 Subject: [PATCH 03/14] Add support for Slack 'triggers' webhooks - streamline webhook handling logic (#90) * Add support for Slack 'triggers' webhooks - streamline webhook handling logic --------- Co-authored-by: Andrew Riley --- handler.py | 60 ++++++++++++++++++--------------------------- messagegenerator.py | 4 +-- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/handler.py b/handler.py index 2be0414..f7eece4 100644 --- a/handler.py +++ b/handler.py @@ -85,42 +85,30 @@ def send_alert(event_details, affected_accounts, affected_entities, event_type): except URLError as e: print("Server connection failed: ", e.reason) pass - if "hooks.slack.com/services" in slack_url: - try: - print("Sending the alert to Slack Webhook Channel") - send_to_slack( - get_message_for_slack( - event_details, - event_type, - affected_accounts, - resources, - slack_webhook="webhook", - ), - slack_url, - ) - except HTTPError as e: - print("Got an error while sending message to Slack: ", e.code, e.reason) - except URLError as e: - print("Server connection failed: ", e.reason) - pass - if "hooks.slack.com/workflows" in slack_url: - try: - print("Sending the alert to Slack Workflows Channel") - send_to_slack( - get_message_for_slack( - event_details, - event_type, - affected_accounts, - resources, - slack_webhook="workflow", - ), - slack_url, - ) - except HTTPError as e: - print("Got an error while sending message to Slack: ", e.code, e.reason) - except URLError as e: - print("Server connection failed: ", e.reason) - pass + #Slack Notification Handling + if slack_url is not "None": + for slack_webhook_type in ["services", "triggers", "workflows"]: + if ("hooks.slack.com/" + slack_webhook_type) in slack_url: + try: + send_to_slack( + get_message_for_slack( + event_details, + event_type, + affected_accounts, + resources, + slack_webhook_type, + ), + slack_url, + ) + break + except HTTPError as e: + print("Got an error while sending message to Slack: ", e.code, e.reason) + except URLError as e: + print("Server connection failed: ", e.reason) + pass + else: + print("Unsupported format in Slack Webhook") + if "office.com/webhook" in teams_url: try: print("Sending the alert to Teams") diff --git a/messagegenerator.py b/messagegenerator.py index 12c619b..634f067 100644 --- a/messagegenerator.py +++ b/messagegenerator.py @@ -11,7 +11,7 @@ def get_message_for_slack(event_details, event_type, affected_accounts, affected_entities, slack_webhook): message = "" summary = "" - if slack_webhook == "webhook": + if slack_webhook == "services": #Handle "Incoming Webhook" webhooks if len(affected_entities) >= 1: affected_entities = "\n".join(affected_entities) if affected_entities == "UNKNOWN": @@ -70,7 +70,7 @@ def get_message_for_slack(event_details, event_type, affected_accounts, affected } ] } - else: + else: #Handle 'workflows' or 'triggers' webhooks if len(affected_entities) >= 1: affected_entities = "\n".join(affected_entities) if affected_entities == "UNKNOWN": From 99b84a4416192bc63a1486a3a3de5528e70c4bd3 Mon Sep 17 00:00:00 2001 From: Andrew Riley Date: Thu, 24 Oct 2024 12:33:23 -0400 Subject: [PATCH 04/14] log the attempt to send Slack message (#91) Co-authored-by: Andrew Riley --- handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/handler.py b/handler.py index f7eece4..03ceece 100644 --- a/handler.py +++ b/handler.py @@ -89,6 +89,7 @@ def send_alert(event_details, affected_accounts, affected_entities, event_type): if slack_url is not "None": for slack_webhook_type in ["services", "triggers", "workflows"]: if ("hooks.slack.com/" + slack_webhook_type) in slack_url: + print("Sending the alert to Slack Webhook Channel") try: send_to_slack( get_message_for_slack( From ecf1b3969d5fad27444f7d3390a5bac88b10a470 Mon Sep 17 00:00:00 2001 From: David Bruce Date: Sat, 26 Oct 2024 08:39:53 +0200 Subject: [PATCH 05/14] Configure python logging --- handler.py | 5 +++++ messagegenerator.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/handler.py b/handler.py index 2be0414..8c0dbf8 100644 --- a/handler.py +++ b/handler.py @@ -1,4 +1,6 @@ import json +import logging + import boto3 import os import re @@ -25,6 +27,9 @@ get_detail_for_eventbridge, ) +logger = logging.getLogger() +logger.setLevel(os.environ.get("LOG_LEVEL", "INFO").upper()) + print("boto3 version: ", boto3.__version__) # query active health API endpoint diff --git a/messagegenerator.py b/messagegenerator.py index 12c619b..ba4975f 100644 --- a/messagegenerator.py +++ b/messagegenerator.py @@ -6,6 +6,9 @@ import re import sys import time +import logging + +logger = logging.getLogger() def get_message_for_slack(event_details, event_type, affected_accounts, affected_entities, slack_webhook): From aae649dfab1d5774c426a1815bd64debd0b6a221 Mon Sep 17 00:00:00 2001 From: David Bruce Date: Sat, 26 Oct 2024 08:43:36 +0200 Subject: [PATCH 06/14] Truncate long chime messages Chime has a 4096 byte limit for sent messages. Truncate any longer messages and indicate that the message is truncated. Fixes #70 --- messagegenerator.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/messagegenerator.py b/messagegenerator.py index ba4975f..59d02f5 100644 --- a/messagegenerator.py +++ b/messagegenerator.py @@ -290,7 +290,7 @@ def get_message_for_chime(event_details, event_type, affected_accounts, affected "**Event ARN**: " + event_details['successfulSet'][0]['event']['arn'] + "\n" "**Updates:**" + "\n" + get_last_aws_update(event_details) ) - json.dumps(message) + message = truncate_message_if_needed(message, 4096) print("Message sent to Chime: ", message) return message @@ -334,6 +334,7 @@ def get_org_message_for_chime(event_details, event_type, affected_org_accounts, "**Event ARN**: " + event_details['successfulSet'][0]['event']['arn'] + "\n" "**Updates:**" + "\n" + get_last_aws_update(event_details) ) + message = truncate_message_if_needed(message, 4096) print("Message sent to Chime: ", message) return message @@ -632,3 +633,21 @@ def format_date(event_time): """ event_time = datetime.strptime(event_time[:16], '%Y-%m-%d %H:%M') return event_time.strftime('%B %d, %Y at %I:%M %p') + + +def truncate_message_if_needed(message, max_length): + """ + Truncates the message if it exceeds the specified maximum length. + + :param message: Message you want to truncate. + :type message: str + :param max_length: Length at which to truncate the message. + :type max_length: int + :return: Possibly truncated message. + :rtype: str + """ + message_length = len(message) + if message_length > max_length: + print(f"Message length of {message_length} is too long, truncating to {max_length}.") + message = message[:(max_length - 3)] + "..." + return message From 5cca8fbddb4146a1bf080ad7945e4eb9efc3e3f2 Mon Sep 17 00:00:00 2001 From: David Bruce Date: Sat, 26 Oct 2024 08:59:51 +0200 Subject: [PATCH 07/14] If endTime is missing set it to "Unknown" in messages Fixes #77 --- messagegenerator.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/messagegenerator.py b/messagegenerator.py index 59d02f5..3a554a9 100644 --- a/messagegenerator.py +++ b/messagegenerator.py @@ -65,7 +65,7 @@ def get_message_for_slack(event_details, event_type, affected_accounts, affected { "title": "Service", "value": event_details['successfulSet'][0]['event']['service'], "short": True }, { "title": "Region", "value": event_details['successfulSet'][0]['event']['region'], "short": True }, { "title": "Start Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['startTime']), "short": True }, - { "title": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['endTime']), "short": True }, + { "title": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event'].get('endTime')), "short": True }, { "title": "Status", "value": event_details['successfulSet'][0]['event']['statusCode'], "short": True }, { "title": "Event ARN", "value": event_details['successfulSet'][0]['event']['arn'], "short": False }, { "title": "Updates", "value": get_last_aws_update(event_details), "short": False } @@ -194,7 +194,7 @@ def get_org_message_for_slack(event_details, event_type, affected_org_accounts, { "title": "Service", "value": event_details['successfulSet'][0]['event']['service'], "short": True }, { "title": "Region", "value": event_details['successfulSet'][0]['event']['region'], "short": True }, { "title": "Start Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['startTime']), "short": True }, - { "title": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['endTime']), "short": True }, + { "title": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event'].get('endTime')), "short": True }, { "title": "Status", "value": event_details['successfulSet'][0]['event']['statusCode'], "short": True }, { "title": "Event ARN", "value": event_details['successfulSet'][0]['event']['arn'], "short": False }, { "title": "Updates", "value": get_last_aws_update(event_details), "short": False } @@ -285,7 +285,7 @@ def get_message_for_chime(event_details, event_type, affected_accounts, affected "**Service**: " + event_details['successfulSet'][0]['event']['service'] + "\n" "**Region**: " + event_details['successfulSet'][0]['event']['region'] + "\n" "**Start Time (UTC)**: " + cleanup_time(event_details['successfulSet'][0]['event']['startTime']) + "\n" - "**End Time (UTC)**: " + cleanup_time(event_details['successfulSet'][0]['event']['endTime']) + "\n" + "**End Time (UTC)**: " + cleanup_time(event_details['successfulSet'][0]['event'].get('endTime')) + "\n" "**Status**: " + event_details['successfulSet'][0]['event']['statusCode'] + "\n" "**Event ARN**: " + event_details['successfulSet'][0]['event']['arn'] + "\n" "**Updates:**" + "\n" + get_last_aws_update(event_details) @@ -329,7 +329,7 @@ def get_org_message_for_chime(event_details, event_type, affected_org_accounts, "**Service**: " + event_details['successfulSet'][0]['event']['service'] + "\n" "**Region**: " + event_details['successfulSet'][0]['event']['region'] + "\n" "**Start Time (UTC)**: " + cleanup_time(event_details['successfulSet'][0]['event']['startTime']) + "\n" - "**End Time (UTC)**: " + cleanup_time(event_details['successfulSet'][0]['event']['endTime']) + "\n" + "**End Time (UTC)**: " + cleanup_time(event_details['successfulSet'][0]['event'].get('endTime')) + "\n" "**Status**: " + event_details['successfulSet'][0]['event']['statusCode'] + "\n" "**Event ARN**: " + event_details['successfulSet'][0]['event']['arn'] + "\n" "**Updates:**" + "\n" + get_last_aws_update(event_details) @@ -399,7 +399,7 @@ def get_message_for_teams(event_details, event_type, affected_accounts, affected {"name": "Service", "value": event_details['successfulSet'][0]['event']['service']}, {"name": "Region", "value": event_details['successfulSet'][0]['event']['region']}, {"name": "Start Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['startTime'])}, - {"name": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['endTime'])}, + {"name": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event'].get('endTime'))}, {"name": "Status", "value": event_details['successfulSet'][0]['event']['statusCode']}, {"name": "Event ARN", "value": event_details['successfulSet'][0]['event']['arn']}, {"name": "Updates", "value": get_last_aws_update(event_details)} @@ -468,7 +468,7 @@ def get_org_message_for_teams(event_details, event_type, affected_org_accounts, {"name": "Service", "value": event_details['successfulSet'][0]['event']['service']}, {"name": "Region", "value": event_details['successfulSet'][0]['event']['region']}, {"name": "Start Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['startTime'])}, - {"name": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event']['endTime'])}, + {"name": "End Time (UTC)", "value": cleanup_time(event_details['successfulSet'][0]['event'].get('endTime'))}, {"name": "Status", "value": event_details['successfulSet'][0]['event']['statusCode']}, {"name": "Event ARN", "value": event_details['successfulSet'][0]['event']['arn']}, {"name": "Updates", "value": event_details['successfulSet'][0]['eventDescription']['latestDescription']} @@ -524,7 +524,7 @@ def get_message_for_email(event_details, event_type, affected_accounts, affected Service: {event_details['successfulSet'][0]['event']['service']}
Region: {event_details['successfulSet'][0]['event']['region']}
Start Time (UTC): {cleanup_time(event_details['successfulSet'][0]['event']['startTime'])}
- End Time (UTC): {cleanup_time(event_details['successfulSet'][0]['event']['endTime'])}
+ End Time (UTC): {cleanup_time(event_details['successfulSet'][0]['event'].get('endTime'))}
Status: {event_details['successfulSet'][0]['event']['statusCode']}
Event ARN: {event_details['successfulSet'][0]['event']['arn']}
Updates: {event_details['successfulSet'][0]['eventDescription']['latestDescription']}

@@ -580,7 +580,7 @@ def get_org_message_for_email(event_details, event_type, affected_org_accounts, Service: {event_details['successfulSet'][0]['event']['service']}
Region: {event_details['successfulSet'][0]['event']['region']}
Start Time (UTC): {cleanup_time(event_details['successfulSet'][0]['event']['startTime'])}
- End Time (UTC): {cleanup_time(event_details['successfulSet'][0]['event']['endTime'])}
+ End Time (UTC): {cleanup_time(event_details['successfulSet'][0]['event'].get('endTime'))}
Status: {event_details['successfulSet'][0]['event']['statusCode']}
Event ARN: {event_details['successfulSet'][0]['event']['arn']}
Updates: {event_details['successfulSet'][0]['eventDescription']['latestDescription']}

@@ -604,6 +604,9 @@ def cleanup_time(event_time): :return: A formatted string that includes the month, date, year and 12-hour time. :rtype: str """ + if not event_time: + return "Unknown" + event_time = datetime.strptime(event_time[:16], '%Y-%m-%d %H:%M') return event_time.strftime("%Y-%m-%d %H:%M:%S") From a6a803a9017585e7eb1e41d8d775090a1cf0678e Mon Sep 17 00:00:00 2001 From: David Bruce Date: Sat, 26 Oct 2024 09:11:40 +0200 Subject: [PATCH 08/14] Remove unused imports --- handler.py | 6 ------ messagegenerator.py | 7 +------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/handler.py b/handler.py index 8c0dbf8..2c04697 100644 --- a/handler.py +++ b/handler.py @@ -3,18 +3,12 @@ import boto3 import os -import re -import time -import decimal import socket -import configparser from dateutil import parser from datetime import datetime, timedelta -from urllib.parse import urlencode from urllib.request import Request, urlopen, URLError, HTTPError from botocore.config import Config from botocore.exceptions import ClientError -from boto3.dynamodb.conditions import Key, Attr from messagegenerator import ( get_message_for_slack, get_org_message_for_slack, diff --git a/messagegenerator.py b/messagegenerator.py index 3a554a9..97b17bb 100644 --- a/messagegenerator.py +++ b/messagegenerator.py @@ -1,11 +1,6 @@ import json -import boto3 -from datetime import datetime, timedelta -from botocore.exceptions import ClientError -import os -import re +from datetime import datetime import sys -import time import logging logger = logging.getLogger() From 6e8e8419c6cb684cc97aea666a23d17604688500 Mon Sep 17 00:00:00 2001 From: David Bruce Date: Sat, 26 Oct 2024 10:25:46 +0200 Subject: [PATCH 09/14] Cache boto3 clients To reduce the number of boto3 clients that are created we introduce AWSApi which uses functools.lru_cache to cache boto3 clients. This can significantly reduce the lambda runtime. --- handler.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/handler.py b/handler.py index 2c04697..994d8f9 100644 --- a/handler.py +++ b/handler.py @@ -1,5 +1,6 @@ import json import logging +from functools import lru_cache import boto3 import os @@ -24,6 +25,21 @@ logger = logging.getLogger() logger.setLevel(os.environ.get("LOG_LEVEL", "INFO").upper()) +class AWSApi: + @lru_cache + def client(self, *args, **kwargs): + logger.debug(f"Returning new boto3 client for: {args}") + return boto3.client(*args, **kwargs) + + @lru_cache + def resource(self, resource_name): + logger.debug(f"Returning new boto3 resource for: {resource_name}") + return boto3.resource(resource_name) + + def cache_clear(self): + self.client.cache_clear() + self.resource.cache_clear() + print("boto3 version: ", boto3.__version__) # query active health API endpoint @@ -43,6 +59,8 @@ ), ) +aws_api = AWSApi() + # TODO decide if account_name should be blank on error # Get Account Name @@ -322,7 +340,7 @@ def send_email(event_details, eventType, affected_accounts, affected_entities): BODY_HTML = get_message_for_email( event_details, eventType, affected_accounts, affected_entities ) - client = boto3.client("ses", region_name=AWS_REGION) + client = aws_api.client("ses", AWS_REGION) response = client.send_email( Source=SENDER, Destination={"ToAddresses": RECIPIENT}, @@ -349,7 +367,7 @@ def send_org_email( BODY_HTML = get_org_message_for_email( event_details, eventType, affected_org_accounts, affected_org_entities ) - client = boto3.client("ses", region_name=AWS_REGION) + client = aws_api.client("ses", AWS_REGION) response = client.send_email( Source=SENDER, Destination={"ToAddresses": RECIPIENT}, @@ -468,7 +486,7 @@ def update_org_ddb( affected_org_entities, ): # open dynamoDB - dynamodb = boto3.resource("dynamodb") + dynamodb = aws_api.resource("dynamodb") ddb_table = os.environ["DYNAMODB_TABLE"] aha_ddb_table = dynamodb.Table(ddb_table) event_latestDescription = event_details["successfulSet"][0]["eventDescription"][ @@ -583,7 +601,7 @@ def update_ddb( affected_entities, ): # open dynamoDB - dynamodb = boto3.resource("dynamodb") + dynamodb = aws_api.resource("dynamodb") ddb_table = os.environ["DYNAMODB_TABLE"] aha_ddb_table = dynamodb.Table(ddb_table) event_latestDescription = event_details["successfulSet"][0]["eventDescription"][ @@ -697,8 +715,7 @@ def get_secrets(): secrets = {} # create a Secrets Manager client - session = boto3.session.Session() - client = session.client(service_name="secretsmanager", region_name=region_name) + client = aws_api.client("secretsmanager", region_name=region_name) # Iteration through the configured AWS Secrets secrets["teams"] = ( get_secret(secret_teams_name, client) if "Teams" in os.environ else "None" @@ -964,7 +981,7 @@ def send_to_eventbridge(message, event_type, resources, event_bus): print( "Sending response to Eventbridge - event_type, event_bus", event_type, event_bus ) - client = boto3.client("events") + client = aws_api.client("events") entries = eventbridge_generate_entries(message, resources, event_bus) @@ -979,7 +996,7 @@ def getAccountIDs(): key_file_name = os.environ["ACCOUNT_IDS"] print("Key filename is - ", key_file_name) if os.path.splitext(os.path.basename(key_file_name))[1] == ".csv": - s3 = boto3.client("s3") + s3 = aws_api.client("s3") data = s3.get_object(Bucket=os.environ["S3_BUCKET"], Key=key_file_name) account_ids = [account.decode("utf-8") for account in data["Body"].iter_lines()] else: @@ -997,7 +1014,7 @@ def get_sts_token(service): SECRET_KEY = [] SESSION_TOKEN = [] - sts_connection = boto3.client("sts") + sts_connection = aws_api.client("sts") ct = datetime.now() role_session_name = "cross_acct_aha_session" @@ -1013,7 +1030,7 @@ def get_sts_token(service): SESSION_TOKEN = acct_b["Credentials"]["SessionToken"] # create service client using the assumed role credentials, e.g. S3 - boto3_client = boto3.client( + boto3_client = aws_api.client( service, config=config, aws_access_key_id=ACCESS_KEY, @@ -1022,13 +1039,14 @@ def get_sts_token(service): ) print("Running in member account deployment mode") else: - boto3_client = boto3.client(service, config=config) + boto3_client = aws_api.client(service, config=config) print("Running in management account deployment mode") return boto3_client def main(event, context): + aws_api.cache_clear() print("THANK YOU FOR CHOOSING AWS HEALTH AWARE!") health_client = get_sts_token("health") org_status = os.environ["ORG_STATUS"] From c2ac41f8d534c7745ed6dc6741ef4029b2430e46 Mon Sep 17 00:00:00 2001 From: David Bruce Date: Sat, 26 Oct 2024 11:25:04 +0200 Subject: [PATCH 10/14] Cache calls to secretsmanager.get_secret_value Reduce the number of times we need to call the AWS api to fetch secrets by caching the response. --- handler.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/handler.py b/handler.py index 994d8f9..300b955 100644 --- a/handler.py +++ b/handler.py @@ -25,6 +25,16 @@ logger = logging.getLogger() logger.setLevel(os.environ.get("LOG_LEVEL", "INFO").upper()) +class CachedSecrets: + def __init__(self, client): + self.client = client + + @lru_cache + def get_secret_value(self, *args, **kwargs): + logger.debug(f"Getting secret {kwargs}") + return self.client.get_secret_value(*args, **kwargs) + + class AWSApi: @lru_cache def client(self, *args, **kwargs): @@ -40,6 +50,12 @@ def cache_clear(self): self.client.cache_clear() self.resource.cache_clear() + @lru_cache + def secretsmanager(self, **kwargs): + client = boto3.client("secretsmanager", **kwargs) + return CachedSecrets(client) + + print("boto3 version: ", boto3.__version__) # query active health API endpoint @@ -715,7 +731,7 @@ def get_secrets(): secrets = {} # create a Secrets Manager client - client = aws_api.client("secretsmanager", region_name=region_name) + client = aws_api.secretsmanager(region_name=region_name) # Iteration through the configured AWS Secrets secrets["teams"] = ( get_secret(secret_teams_name, client) if "Teams" in os.environ else "None" From f7e3496b3795653cc7034080a9d1de1033aa70b0 Mon Sep 17 00:00:00 2001 From: Andrew Riley Date: Wed, 30 Oct 2024 12:29:06 -0400 Subject: [PATCH 11/14] update to Slack webhook/trigger handling --- handler.py | 63 ++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/handler.py b/handler.py index 03ceece..14797c1 100644 --- a/handler.py +++ b/handler.py @@ -86,7 +86,7 @@ def send_alert(event_details, affected_accounts, affected_entities, event_type): print("Server connection failed: ", e.reason) pass #Slack Notification Handling - if slack_url is not "None": + if slack_url != "None": for slack_webhook_type in ["services", "triggers", "workflows"]: if ("hooks.slack.com/" + slack_webhook_type) in slack_url: print("Sending the alert to Slack Webhook Channel") @@ -179,42 +179,31 @@ def send_org_alert( except URLError as e: print("Server connection failed: ", e.reason) pass - if "hooks.slack.com/services" in slack_url: - try: - print("Sending the alert to Slack Webhook Channel") - send_to_slack( - get_org_message_for_slack( - event_details, - event_type, - affected_org_accounts, - resources, - slack_webhook="webhook", - ), - slack_url, - ) - except HTTPError as e: - print("Got an error while sending message to Slack: ", e.code, e.reason) - except URLError as e: - print("Server connection failed: ", e.reason) - pass - if "hooks.slack.com/workflows" in slack_url: - try: - print("Sending the alert to Slack Workflow Channel") - send_to_slack( - get_org_message_for_slack( - event_details, - event_type, - affected_org_accounts, - resources, - slack_webhook="workflow", - ), - slack_url, - ) - except HTTPError as e: - print("Got an error while sending message to Slack: ", e.code, e.reason) - except URLError as e: - print("Server connection failed: ", e.reason) - pass + #Slack Notification Handling + if slack_url != "None": + for slack_webhook_type in ["services", "triggers", "workflows"]: + if ("hooks.slack.com/" + slack_webhook_type) in slack_url: + print("Sending the alert to Slack Webhook Channel") + try: + send_to_slack( + get_message_for_slack( + event_details, + event_type, + affected_org_accounts, + resources, + slack_webhook_type, + ), + slack_url, + ) + break + except HTTPError as e: + print("Got an error while sending message to Slack: ", e.code, e.reason) + except URLError as e: + print("Server connection failed: ", e.reason) + pass + else: + print("Unsupported format in Slack Webhook") + if "office.com/webhook" in teams_url: try: print("Sending the alert to Teams") From c3da096a14bd9597bb4d4d7ae13c2cd1f990923a Mon Sep 17 00:00:00 2001 From: Andrew Riley Date: Thu, 7 Nov 2024 13:32:39 -0500 Subject: [PATCH 12/14] add sample ExcludeAccountIDs file --- ExcludeAccountIDs(sample).csv | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ExcludeAccountIDs(sample).csv diff --git a/ExcludeAccountIDs(sample).csv b/ExcludeAccountIDs(sample).csv new file mode 100644 index 0000000..feb3a97 --- /dev/null +++ b/ExcludeAccountIDs(sample).csv @@ -0,0 +1,2 @@ +000000000000 +111111111111 From ccd1396d85b7fa0b0bc10daa0723bad896897588 Mon Sep 17 00:00:00 2001 From: Andrew Riley Date: Thu, 7 Nov 2024 14:02:22 -0500 Subject: [PATCH 13/14] add troubleshooting info for ExcludeAccountIDs file --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5c7e7cb..1265a1f 100644 --- a/README.md +++ b/README.md @@ -462,6 +462,8 @@ We are happy to announce the launch of new enhancements to AHA. Please try them * If for whatever reason you need to update the Webhook URL; just update the CloudFormation or terraform Template with the new Webhook URL. * If you are expecting an event and it did not show up it may be an oddly formed event. Take a look at *CloudWatch > Log groups* and search for the name of your Lambda function. See what the error is and reach out to us [email](mailto:aha-builders@amazon.com) for help. * If for any errors related to duplicate secrets during deployment, try deleting manually and redeploy the solution. Example command to delete SlackChannelID secret in us-east-1 region. -``` -$ aws secretsmanager delete-secret --secret-id SlackChannelID --force-delete-without-recovery --region us-east-1 -``` + ``` + $ aws secretsmanager delete-secret --secret-id SlackChannelID --force-delete-without-recovery --region us-east-1 + ``` +* If you want to Exclude certain accounts from notifications, confirm your exlcusions file matches the format of the [sample ExcludeAccountIDs.csv file](ExcludeAccountIDs(sample).csv) with one account ID per line with no trailing commas (trailing commas indicate a null cell). + * If your accounts listed in the CSV file are not excluded, check the CloudWatch log group for the AHA Lambda function for the message "Key filename is not a .csv file" as an indicator of any issues with your file. From 9814f5872760d0e803be843637d719802c721dea Mon Sep 17 00:00:00 2001 From: Andrew Riley Date: Mon, 11 Nov 2024 10:46:47 -0500 Subject: [PATCH 14/14] Add release 2.3 notes --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1265a1f..9932004 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,7 @@ AWS Health Aware (AHA) is an automated notification tool for sending well-format # What's New -Release 2.2 introduces an updated schema for Health events delivered to an EventBridge bus. This allows simplified matching of events which you can then consume with other AWS services or SaaS solutions. -Read more about the [new feature and how to filter events using EventBridge](https://github.com/aws-samples/aws-health-aware/blob/main/new_aha_event_schema.md). +Release 2.3 introduces runtime performance improvements, terraform updates, allows use of Slack Workflow 2.0 webhooks (triggers), general fixes and documentation updates. # Architecture @@ -447,6 +446,8 @@ $ terraform apply **If for some reason, you still have issues after updating, you can easily just delete the stack and redeploy. The infrastructure can be destroyed and rebuilt within minutes through Terraform.** # New Features +*Release 2.2* + We are happy to announce the launch of new enhancements to AHA. Please try them out and keep sending us your feedback! 1. A revised schema for AHA events sent to EventBridge which enables new filtering and routing options. See the [new AHA event schema readme](new_aha_event_schema.md) for more detail. 2. Multi-region deployment option