From 2903766ef81d18f903a4eae939839d15b5e52b36 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 09:18:51 +0000 Subject: [PATCH 01/19] change file name --- .../region/modules/event_bus/bus.tf | 59 +++++++++++ .../region/modules/event_bus/main.tf | 100 ------------------ 2 files changed, 59 insertions(+), 100 deletions(-) create mode 100644 terraform/environment/region/modules/event_bus/bus.tf delete mode 100644 terraform/environment/region/modules/event_bus/main.tf diff --git a/terraform/environment/region/modules/event_bus/bus.tf b/terraform/environment/region/modules/event_bus/bus.tf new file mode 100644 index 0000000000..30f557ed14 --- /dev/null +++ b/terraform/environment/region/modules/event_bus/bus.tf @@ -0,0 +1,59 @@ +resource "aws_cloudwatch_event_bus" "main" { + count = var.event_bus_enabled ? 1 : 0 + name = var.environment_name + provider = aws.region +} + +resource "aws_cloudwatch_event_archive" "main" { + count = var.event_bus_enabled ? 1 : 0 + name = var.environment_name + event_source_arn = aws_cloudwatch_event_bus.main[0].arn + provider = aws.region +} + + +resource "aws_cloudwatch_event_rule" "receive_events_from_mlpa" { + count = var.event_bus_enabled ? 1 : 0 + name = "${var.environment_name}-mlpa-events-to-use" + description = "Receive events from mlpa" + event_bus_name = aws_cloudwatch_event_bus.main[0].name + + event_pattern = jsonencode({ + source = ["opg.poas.makeregister"], + detail-type = ["lpa-access-granted"] + }) + + provider = aws.region +} + +resource "aws_cloudwatch_event_bus_policy" "cross_account_receive" { + count = length(var.receive_account_ids) > 0 ? 1 : 0 + event_bus_name = aws_cloudwatch_event_bus.main.name + policy = data.aws_iam_policy_document.cross_account_receive.json + provider = aws.region +} + +# Allow MLPA account to send messages +data "aws_iam_policy_document" "cross_account_receive" { + statement { + sid = "CrossAccountAccess" + effect = "Allow" + actions = [ + "events:PutEvents", + ] + resources = [ + aws_cloudwatch_event_bus.main.arn + ] + + principals { + type = "AWS" + identifiers = var.receive_account_ids + } + } +} + +resource "aws_cloudwatch_event_target" "receive_events" { + count = var.event_bus_enabled ? 1 : 0 + rule = aws_cloudwatch_event_rule.receive_events_from_mlpa[0].name + arn = aws_sqs_queue.receive_events_queue.arn +} diff --git a/terraform/environment/region/modules/event_bus/main.tf b/terraform/environment/region/modules/event_bus/main.tf deleted file mode 100644 index 8e8711d41e..0000000000 --- a/terraform/environment/region/modules/event_bus/main.tf +++ /dev/null @@ -1,100 +0,0 @@ -resource "aws_cloudwatch_event_bus" "main" { - count = var.event_bus_enabled ? 1 : 0 - name = var.environment_name - provider = aws.region -} - -resource "aws_cloudwatch_event_archive" "main" { - count = var.event_bus_enabled ? 1 : 0 - name = var.environment_name - event_source_arn = aws_cloudwatch_event_bus.main[0].arn - provider = aws.region -} - -resource "aws_cloudwatch_event_rule" "receive_events_mlpa" { - count = var.event_bus_enabled ? 1 : 0 - name = "${var.environment_name}-mlpa-events-to-use" - description = "receive events from mlpa" - event_bus_name = aws_cloudwatch_event_bus.main[0].name - - event_pattern = jsonencode({ - source = ["opg.poas.makeregister"], - }) - provider = aws.region -} - -data "aws_kms_alias" "sqs" { - name = "alias/sqs-mrk" - provider = aws.region -} - -resource "aws_sqs_queue" "receive_events_queue" { - count = var.event_bus_enabled ? 1 : 0 - name = "${var.environment_name}-receive-events-queue" - kms_master_key_id = data.aws_kms_alias.sqs.target_key_id - kms_data_key_reuse_period_seconds = 300 - - visibility_timeout_seconds = 300 - - redrive_policy = jsonencode({ - deadLetterTargetArn = aws_sqs_queue.receive_events_deadletter[0].arn - maxReceiveCount = 3 - }) - policy = data.aws_iam_policy_document.receive_events_queue_policy[0].json - - provider = aws.region -} - -data "aws_iam_policy_document" "receive_events_queue_policy" { - count = var.event_bus_enabled ? 1 : 0 - statement { - sid = "${var.current_region}-ReceiveFromMLPA" - effect = "Allow" - - principals { - type = "Service" - identifiers = ["events.amazonaws.com"] - } - - actions = ["sqs:SendMessage"] - resources = ["*"] - - condition { - test = "ArnEquals" - variable = "aws:SourceArn" - values = [ - aws_cloudwatch_event_rule.receive_events_mlpa[0].arn - ] - } - } -} - -resource "aws_sqs_queue" "receive_events_deadletter" { - count = var.event_bus_enabled ? 1 : 0 - name = "${var.environment_name}-receive-events-deadletter" - kms_master_key_id = data.aws_kms_alias.sqs.target_key_id - kms_data_key_reuse_period_seconds = 300 - provider = aws.region -} - -resource "aws_sqs_queue_redrive_allow_policy" "receive_events_redrive_allow_policy" { - count = var.event_bus_enabled ? 1 : 0 - queue_url = aws_sqs_queue.receive_events_deadletter[0].id - - redrive_allow_policy = jsonencode({ - redrivePermission = "byQueue", - sourceQueueArns = [aws_sqs_queue.receive_events_queue[0].arn] - }) - provider = aws.region -} - -/* -resource "aws_lambda_event_source_mapping" "reveive_events_mapping" { - count = var.event_bus_enabled ? 1 : 0 - event_source_arn = aws_sqs_queue.receive_events_queue[0].arn - enabled = false - function_name = var.ingress_lambda_name - batch_size = 10 - provider = aws.region -} -*/ From 85c40bb06a62d00b1565a276f74fd8a56984b3c5 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 09:19:57 +0000 Subject: [PATCH 02/19] add sqs and variables --- .../region/modules/event_bus/sqs.tf | 73 +++++++++++++++++++ .../region/modules/event_bus/variables.tf | 9 ++- 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 terraform/environment/region/modules/event_bus/sqs.tf diff --git a/terraform/environment/region/modules/event_bus/sqs.tf b/terraform/environment/region/modules/event_bus/sqs.tf new file mode 100644 index 0000000000..3fe5737194 --- /dev/null +++ b/terraform/environment/region/modules/event_bus/sqs.tf @@ -0,0 +1,73 @@ +data "aws_kms_alias" "sqs" { + name = "alias/sqs-mrk" + provider = aws.region +} + +resource "aws_sqs_queue" "receive_events_queue" { + count = var.event_bus_enabled ? 1 : 0 + name = "${var.environment_name}-receive-events-queue" + kms_master_key_id = data.aws_kms_alias.sqs.target_key_id + kms_data_key_reuse_period_seconds = 300 + + visibility_timeout_seconds = 300 + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.receive_events_deadletter[0].arn + maxReceiveCount = 3 + }) + + policy = data.aws_iam_policy_document.receive_events_queue_policy[0].json + + provider = aws.region +} + +resource "aws_sqs_queue" "receive_events_deadletter" { + count = var.event_bus_enabled ? 1 : 0 + name = "${var.environment_name}-receive-events-deadletter" + kms_master_key_id = data.aws_kms_alias.sqs.target_key_id + kms_data_key_reuse_period_seconds = 300 + provider = aws.region +} + +resource "aws_sqs_queue_redrive_allow_policy" "receive_events_redrive_allow_policy" { + count = var.event_bus_enabled ? 1 : 0 + queue_url = aws_sqs_queue.receive_events_deadletter[0].id + + redrive_allow_policy = jsonencode({ + redrivePermission = "byQueue", + sourceQueueArns = [aws_sqs_queue.receive_events_queue[0].arn] + }) + provider = aws.region +} + +data "aws_iam_policy_document" "receive_events_queue_policy" { + count = var.event_bus_enabled ? 1 : 0 + statement { + sid = "${var.current_region}-ReceiveFromMLPA" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + actions = ["sqs:SendMessage"] + resources = ["*"] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [ + aws_cloudwatch_event_rule.receive_events_from_mlpa[0].arn + ] + } + } +} + +resource "aws_lambda_event_source_mapping" "receive_events_mapping" { + count = var.event_bus_enabled ? 1 : 0 + event_source_arn = aws_sqs_queue.receive_events_queue.arn + function_name = var.lambda_function_name + enabled = true + provider = aws.region +} diff --git a/terraform/environment/region/modules/event_bus/variables.tf b/terraform/environment/region/modules/event_bus/variables.tf index c1bb1610a2..4ee3c7d48a 100644 --- a/terraform/environment/region/modules/event_bus/variables.tf +++ b/terraform/environment/region/modules/event_bus/variables.tf @@ -9,14 +9,17 @@ variable "event_bus_enabled" { default = false } -/* -variable "ingress_lambda_name" { +variable "lambda_function_name" { description = "The name of the ingress lambda" type = string } -*/ variable "current_region" { description = "The current region" type = string } + +variable "receive_account_ids" { + description = "The account ids that can send events to the event bus" + type = list(string) +} From fe2060b1e3805aa47885c9a9b6666478b820c25d Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 10:38:33 +0000 Subject: [PATCH 03/19] update variable names --- terraform/environment/modules/lambda/outputs.tf | 5 +++++ terraform/environment/region.tf | 4 ++++ terraform/environment/region/event_bus.tf | 10 ++++++---- .../environment/region/modules/event_bus/bus.tf | 6 +++--- .../environment/region/modules/event_bus/sqs.tf | 4 ++-- .../region/modules/event_bus/variables.tf | 5 +++-- terraform/environment/region/variables.tf | 11 +++++++++++ terraform/environment/terraform.tfvars.json | 15 +++++++++++++++ terraform/environment/variables.tf | 1 + 9 files changed, 50 insertions(+), 11 deletions(-) diff --git a/terraform/environment/modules/lambda/outputs.tf b/terraform/environment/modules/lambda/outputs.tf index 9b6894875d..2bb0c98882 100644 --- a/terraform/environment/modules/lambda/outputs.tf +++ b/terraform/environment/modules/lambda/outputs.tf @@ -12,3 +12,8 @@ output "lambda_role" { description = "The lambda role" value = aws_iam_role.lambda_role } + +output "lambda_name" { + description = "The lambda name" + value = aws_lambda_function.lambda_function.function_name +} diff --git a/terraform/environment/region.tf b/terraform/environment/region.tf index 1dffca588e..94ae5d4f65 100644 --- a/terraform/environment/region.tf +++ b/terraform/environment/region.tf @@ -20,6 +20,7 @@ module "eu_west_1" { ecs_task_roles = module.iam.ecs_task_roles environment_name = local.environment_name event_bus_enabled = local.environment.event_bus_enabled + event_receiver_lambda_name = module.event_receiver.lambda_name google_analytics_id_use = local.environment.google_analytics_id_use google_analytics_id_view = local.environment.google_analytics_id_view gov_uk_onelogin_client_id_secret_name = local.environment.gov_uk_onelogin_client_id_secret_name @@ -33,6 +34,7 @@ module "eu_west_1" { lpa_codes_endpoint = local.environment.lpa_codes_endpoint lpas_collection_endpoint = local.environment.lpas_collection_endpoint lpa_data_store_endpoint = local.environment.lpa_data_store_endpoint + receive_account_ids = local.environment.receive_account_ids mock_onelogin_enabled = local.environment.mock_onelogin_enabled mock_onelogin_service_container_version = local.mock_onelogin_version mock_onelogin_service_repository_url = data.aws_ecr_repository.mock_onelogin.repository_url @@ -107,6 +109,7 @@ module "eu_west_2" { ecs_task_roles = module.iam.ecs_task_roles environment_name = local.environment_name event_bus_enabled = local.environment.event_bus_enabled + event_receiver_lambda_name = module.event_receiver.lambda_name google_analytics_id_use = local.environment.google_analytics_id_use google_analytics_id_view = local.environment.google_analytics_id_view gov_uk_onelogin_client_id_secret_name = local.environment.gov_uk_onelogin_client_id_secret_name @@ -120,6 +123,7 @@ module "eu_west_2" { lpa_codes_endpoint = local.environment.lpa_codes_endpoint lpas_collection_endpoint = local.environment.lpas_collection_endpoint lpa_data_store_endpoint = local.environment.lpa_data_store_endpoint + receive_account_ids = local.environment.receive_account_ids mock_onelogin_enabled = local.environment.mock_onelogin_enabled mock_onelogin_service_container_version = local.mock_onelogin_version mock_onelogin_service_repository_url = data.aws_ecr_repository.mock_onelogin.repository_url diff --git a/terraform/environment/region/event_bus.tf b/terraform/environment/region/event_bus.tf index 264fe0b882..5ef9c8947a 100644 --- a/terraform/environment/region/event_bus.tf +++ b/terraform/environment/region/event_bus.tf @@ -1,8 +1,10 @@ module "event_bus" { - source = "./modules/event_bus" - environment_name = var.environment_name - event_bus_enabled = var.event_bus_enabled - current_region = data.aws_region.current.name + source = "./modules/event_bus" + environment_name = var.environment_name + event_bus_enabled = var.event_bus_enabled + current_region = data.aws_region.current.name + receive_account_ids = var.receive_account_ids + event_receiver_lambda_name = var.event_receiver_lambda_name providers = { aws.region = aws.region } diff --git a/terraform/environment/region/modules/event_bus/bus.tf b/terraform/environment/region/modules/event_bus/bus.tf index 30f557ed14..626070fd05 100644 --- a/terraform/environment/region/modules/event_bus/bus.tf +++ b/terraform/environment/region/modules/event_bus/bus.tf @@ -28,7 +28,7 @@ resource "aws_cloudwatch_event_rule" "receive_events_from_mlpa" { resource "aws_cloudwatch_event_bus_policy" "cross_account_receive" { count = length(var.receive_account_ids) > 0 ? 1 : 0 - event_bus_name = aws_cloudwatch_event_bus.main.name + event_bus_name = aws_cloudwatch_event_bus.main[0].name policy = data.aws_iam_policy_document.cross_account_receive.json provider = aws.region } @@ -42,7 +42,7 @@ data "aws_iam_policy_document" "cross_account_receive" { "events:PutEvents", ] resources = [ - aws_cloudwatch_event_bus.main.arn + aws_cloudwatch_event_bus.main[0].arn ] principals { @@ -55,5 +55,5 @@ data "aws_iam_policy_document" "cross_account_receive" { resource "aws_cloudwatch_event_target" "receive_events" { count = var.event_bus_enabled ? 1 : 0 rule = aws_cloudwatch_event_rule.receive_events_from_mlpa[0].name - arn = aws_sqs_queue.receive_events_queue.arn + arn = aws_sqs_queue.receive_events_queue[0].arn } diff --git a/terraform/environment/region/modules/event_bus/sqs.tf b/terraform/environment/region/modules/event_bus/sqs.tf index 3fe5737194..56009ab6e8 100644 --- a/terraform/environment/region/modules/event_bus/sqs.tf +++ b/terraform/environment/region/modules/event_bus/sqs.tf @@ -66,8 +66,8 @@ data "aws_iam_policy_document" "receive_events_queue_policy" { resource "aws_lambda_event_source_mapping" "receive_events_mapping" { count = var.event_bus_enabled ? 1 : 0 - event_source_arn = aws_sqs_queue.receive_events_queue.arn - function_name = var.lambda_function_name + event_source_arn = aws_sqs_queue.receive_events_queue[0].arn + function_name = var.event_receiver_lambda_name enabled = true provider = aws.region } diff --git a/terraform/environment/region/modules/event_bus/variables.tf b/terraform/environment/region/modules/event_bus/variables.tf index 4ee3c7d48a..1ae7cad034 100644 --- a/terraform/environment/region/modules/event_bus/variables.tf +++ b/terraform/environment/region/modules/event_bus/variables.tf @@ -9,11 +9,12 @@ variable "event_bus_enabled" { default = false } -variable "lambda_function_name" { - description = "The name of the ingress lambda" +variable "event_receiver_lambda_name" { + description = "The name of the ingress from MLPA lambda" type = string } + variable "current_region" { description = "The current region" type = string diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf index a1bf332c49..19ec453163 100644 --- a/terraform/environment/region/variables.tf +++ b/terraform/environment/region/variables.tf @@ -112,6 +112,12 @@ variable "event_bus_enabled" { default = false } +variable "event_receiver_lambda_name" { + description = "The name of the event receiver lambda." + type = string + default = "" +} + variable "feature_flags" { description = "The feature flags to use." type = map(string) @@ -183,6 +189,11 @@ variable "load_balancer_deletion_protection_enabled" { default = false } +variable "receive_account_ids" { + description = "The account ID of the MLPA account." + type = list(string) +} + variable "mock_onelogin_enabled" { description = "Whether or not to enable the mock One Login service." type = bool diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json index d8fc24c215..c271c0f5fe 100644 --- a/terraform/environment/terraform.tfvars.json +++ b/terraform/environment/terraform.tfvars.json @@ -50,6 +50,9 @@ "session_expiry_warning": 5, "ship_metrics_queue_enabled": true, "sirius_account_id": "288342028542", + "receive_account_ids": [ + "653761790766" + ], "load_balancer_deletion_protection_enabled": false, "notify_key_secret_name": "notify-api-key", "associate_alb_with_waf_web_acl_enabled": false, @@ -148,6 +151,9 @@ "session_expiry_warning": 5, "ship_metrics_queue_enabled": true, "sirius_account_id": "288342028542", + "receive_account_ids": [ + "653761790766" + ], "load_balancer_deletion_protection_enabled": false, "notify_key_secret_name": "notify-api-key-demo", "associate_alb_with_waf_web_acl_enabled": true, @@ -246,6 +252,9 @@ "session_expiry_warning": 5, "ship_metrics_queue_enabled": true, "sirius_account_id": "288342028542", + "receive_account_ids": [ + "653761790766" + ], "load_balancer_deletion_protection_enabled": false, "notify_key_secret_name": "notify-api-key-demo", "associate_alb_with_waf_web_acl_enabled": false, @@ -344,6 +353,9 @@ "session_expiry_warning": 5, "ship_metrics_queue_enabled": false, "sirius_account_id": "288342028542", + "receive_account_ids": [ + "653761790766" + ], "load_balancer_deletion_protection_enabled": true, "notify_key_secret_name": "notify-api-key", "associate_alb_with_waf_web_acl_enabled": true, @@ -442,6 +454,9 @@ "session_expiry_warning": 5, "ship_metrics_queue_enabled": false, "sirius_account_id": "649098267436", + "receive_account_ids": [ + "653761790766" + ], "load_balancer_deletion_protection_enabled": true, "notify_key_secret_name": "notify-api-key", "associate_alb_with_waf_web_acl_enabled": true, diff --git a/terraform/environment/variables.tf b/terraform/environment/variables.tf index 0c006272c0..c3f372a588 100644 --- a/terraform/environment/variables.tf +++ b/terraform/environment/variables.tf @@ -73,6 +73,7 @@ variable "environments" { session_expiry_warning = number ship_metrics_queue_enabled = bool sirius_account_id = string + receive_account_ids = list(string) load_balancer_deletion_protection_enabled = bool notify_key_secret_name = string associate_alb_with_waf_web_acl_enabled = bool From 971371d857c7b3acd18e024f3f249f56e5655b20 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 10:46:54 +0000 Subject: [PATCH 04/19] instance key and && operator --- terraform/environment/region/modules/event_bus/bus.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/environment/region/modules/event_bus/bus.tf b/terraform/environment/region/modules/event_bus/bus.tf index 626070fd05..512d5ac59e 100644 --- a/terraform/environment/region/modules/event_bus/bus.tf +++ b/terraform/environment/region/modules/event_bus/bus.tf @@ -29,12 +29,13 @@ resource "aws_cloudwatch_event_rule" "receive_events_from_mlpa" { resource "aws_cloudwatch_event_bus_policy" "cross_account_receive" { count = length(var.receive_account_ids) > 0 ? 1 : 0 event_bus_name = aws_cloudwatch_event_bus.main[0].name - policy = data.aws_iam_policy_document.cross_account_receive.json + policy = data.aws_iam_policy_document.cross_account_receive[0].json provider = aws.region } # Allow MLPA account to send messages data "aws_iam_policy_document" "cross_account_receive" { + count = var.event_bus_enabled ? 1 : 0 statement { sid = "CrossAccountAccess" effect = "Allow" From 58426c5cfe2534ed0c5b6e2be0ff3f875ff7cc68 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 11:15:19 +0000 Subject: [PATCH 05/19] add event target bus name --- .../environment/region/modules/event_bus/bus.tf | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/terraform/environment/region/modules/event_bus/bus.tf b/terraform/environment/region/modules/event_bus/bus.tf index 512d5ac59e..3ccac565f0 100644 --- a/terraform/environment/region/modules/event_bus/bus.tf +++ b/terraform/environment/region/modules/event_bus/bus.tf @@ -27,7 +27,7 @@ resource "aws_cloudwatch_event_rule" "receive_events_from_mlpa" { } resource "aws_cloudwatch_event_bus_policy" "cross_account_receive" { - count = length(var.receive_account_ids) > 0 ? 1 : 0 + count = length(var.receive_account_ids) > 0 && var.event_bus_enabled ? 1 : 0 event_bus_name = aws_cloudwatch_event_bus.main[0].name policy = data.aws_iam_policy_document.cross_account_receive[0].json provider = aws.region @@ -54,7 +54,13 @@ data "aws_iam_policy_document" "cross_account_receive" { } resource "aws_cloudwatch_event_target" "receive_events" { - count = var.event_bus_enabled ? 1 : 0 - rule = aws_cloudwatch_event_rule.receive_events_from_mlpa[0].name - arn = aws_sqs_queue.receive_events_queue[0].arn + count = var.event_bus_enabled ? 1 : 0 + rule = aws_cloudwatch_event_rule.receive_events_from_mlpa[0].name + arn = aws_sqs_queue.receive_events_queue[0].arn + event_bus_name = aws_cloudwatch_event_bus.main[0].name + dead_letter_config { + arn = aws_sqs_queue.receive_events_deadletter[0].arn + } + + provider = aws.region } From 8d721bff5bd9f4fc4a424b5520acc16617a48002 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 16:15:26 +0000 Subject: [PATCH 06/19] allow lambda messages from sqs --- terraform/environment/lambda.tf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index ea0f2c21fa..7916cc3935 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -102,3 +102,23 @@ module "event_receiver" { timeout = 900 memory = 128 } + +resource "aws_iam_role_policy" "lambda_event_receiver" { + name = "${local.environment_name}-lambda-event-receiver" + role = module.event_receiver.lambda_role.id + policy = data.aws_iam_policy_document.lambda_event_receiver.json +} + + +data "aws_iam_policy_document" "lambda_event_receiver" { + statement { + sid = "${local.environment_name}EventReceiverSQS" + effect = "Allow" + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ] + resources = [module.eu_west_1[0].event_bus_sqs_queue_name[0]] + } +} From 68c04a2bc78c13e03f9c478008c32594089ebae0 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Wed, 18 Dec 2024 16:18:47 +0000 Subject: [PATCH 07/19] add vars and outputs --- terraform/environment/region/modules/event_bus/outputs.tf | 4 ++++ terraform/environment/region/outputs.tf | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 terraform/environment/region/modules/event_bus/outputs.tf diff --git a/terraform/environment/region/modules/event_bus/outputs.tf b/terraform/environment/region/modules/event_bus/outputs.tf new file mode 100644 index 0000000000..02063bb0ff --- /dev/null +++ b/terraform/environment/region/modules/event_bus/outputs.tf @@ -0,0 +1,4 @@ +output "receive_events_sqs_queue_name" { + description = "The name of the SQS queue created by the event_bus module." + value = aws_sqs_queue.receive_events_queue[*].name +} diff --git a/terraform/environment/region/outputs.tf b/terraform/environment/region/outputs.tf index a413009a61..685bead097 100644 --- a/terraform/environment/region/outputs.tf +++ b/terraform/environment/region/outputs.tf @@ -45,3 +45,8 @@ output "route53_fqdns" { mock_onelogin = local.route53_fqdns.mock_onelogin } } + +output "event_bus_sqs_queue_name" { + description = "SQS queue name from the event_bus module" + value = module.event_bus.receive_events_sqs_queue_name +} From 511f0233b6e65c4f10f63c7b4ef669d4e5188f2a Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Thu, 19 Dec 2024 08:51:32 +0000 Subject: [PATCH 08/19] use arn instead of id --- terraform/environment/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index 7916cc3935..c38a926db6 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -105,7 +105,7 @@ module "event_receiver" { resource "aws_iam_role_policy" "lambda_event_receiver" { name = "${local.environment_name}-lambda-event-receiver" - role = module.event_receiver.lambda_role.id + role = module.event_receiver.lambda_role.arn policy = data.aws_iam_policy_document.lambda_event_receiver.json } From ca91de13351c21a54d909fdbb2a9d53e6f67c372 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Thu, 19 Dec 2024 09:03:10 +0000 Subject: [PATCH 09/19] just use name --- terraform/environment/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index c38a926db6..8453ffb3ea 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -105,7 +105,7 @@ module "event_receiver" { resource "aws_iam_role_policy" "lambda_event_receiver" { name = "${local.environment_name}-lambda-event-receiver" - role = module.event_receiver.lambda_role.arn + role = module.event_receiver.lambda_role policy = data.aws_iam_policy_document.lambda_event_receiver.json } From 66480b208b1f89905a7e24be93f033cf83e44df5 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Thu, 19 Dec 2024 09:21:51 +0000 Subject: [PATCH 10/19] propagate outputs --- terraform/environment/lambda.tf | 10 ++++++++-- .../environment/region/modules/event_bus/outputs.tf | 10 ++++++++++ .../environment/region/modules/event_bus/sqs.tf | 8 -------- .../region/modules/event_bus/variables.tf | 6 ------ terraform/environment/region/outputs.tf | 12 +++++++++++- terraform/environment/region/variables.tf | 6 ------ 6 files changed, 29 insertions(+), 23 deletions(-) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index 8453ffb3ea..0b0e756719 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -105,7 +105,7 @@ module "event_receiver" { resource "aws_iam_role_policy" "lambda_event_receiver" { name = "${local.environment_name}-lambda-event-receiver" - role = module.event_receiver.lambda_role + role = module.event_receiver.lambda_role.name policy = data.aws_iam_policy_document.lambda_event_receiver.json } @@ -119,6 +119,12 @@ data "aws_iam_policy_document" "lambda_event_receiver" { "sqs:DeleteMessage", "sqs:GetQueueAttributes" ] - resources = [module.eu_west_1[0].event_bus_sqs_queue_name[0]] + resources = [module.eu_west_1[0].receive_events_sqs_queue_name[0]] } } + +resource "aws_lambda_event_source_mapping" "receive_events_mapping" { + event_source_arn = module.eu_west_1[0].receive_events_sqs_queue_arn[0] + function_name = module.event_receiver.lambda_name + enabled = true +} diff --git a/terraform/environment/region/modules/event_bus/outputs.tf b/terraform/environment/region/modules/event_bus/outputs.tf index 02063bb0ff..3c07ec78b6 100644 --- a/terraform/environment/region/modules/event_bus/outputs.tf +++ b/terraform/environment/region/modules/event_bus/outputs.tf @@ -2,3 +2,13 @@ output "receive_events_sqs_queue_name" { description = "The name of the SQS queue created by the event_bus module." value = aws_sqs_queue.receive_events_queue[*].name } + +output "receive_events_sqs_queue_arn" { + description = "The name of the SQS queue created by the event_bus module." + value = aws_sqs_queue.receive_events_queue[*].arn +} + +output "receive_events_bus_arn" { + description = "The ARN of the event bus created by the event_bus module." + value = aws_cloudwatch_event_bus.main[0].arn +} diff --git a/terraform/environment/region/modules/event_bus/sqs.tf b/terraform/environment/region/modules/event_bus/sqs.tf index 56009ab6e8..976e8eddef 100644 --- a/terraform/environment/region/modules/event_bus/sqs.tf +++ b/terraform/environment/region/modules/event_bus/sqs.tf @@ -63,11 +63,3 @@ data "aws_iam_policy_document" "receive_events_queue_policy" { } } } - -resource "aws_lambda_event_source_mapping" "receive_events_mapping" { - count = var.event_bus_enabled ? 1 : 0 - event_source_arn = aws_sqs_queue.receive_events_queue[0].arn - function_name = var.event_receiver_lambda_name - enabled = true - provider = aws.region -} diff --git a/terraform/environment/region/modules/event_bus/variables.tf b/terraform/environment/region/modules/event_bus/variables.tf index 1ae7cad034..1c22663c67 100644 --- a/terraform/environment/region/modules/event_bus/variables.tf +++ b/terraform/environment/region/modules/event_bus/variables.tf @@ -9,12 +9,6 @@ variable "event_bus_enabled" { default = false } -variable "event_receiver_lambda_name" { - description = "The name of the ingress from MLPA lambda" - type = string -} - - variable "current_region" { description = "The current region" type = string diff --git a/terraform/environment/region/outputs.tf b/terraform/environment/region/outputs.tf index 685bead097..7f7cf3357c 100644 --- a/terraform/environment/region/outputs.tf +++ b/terraform/environment/region/outputs.tf @@ -46,7 +46,17 @@ output "route53_fqdns" { } } -output "event_bus_sqs_queue_name" { + +output "receive_events_bus_arn" { + description = "The ARN of the event bus created by the event_bus module." + value = module.event_bus.receive_events_bus_arn +} + +output "receive_events_sqs_queue_arn" { + description = "The name of the SQS queue created by the event_bus module." + value = module.event_bus.receive_events_sqs_queue_arn +} +output "receive_events_sqs_queue_name" { description = "SQS queue name from the event_bus module" value = module.event_bus.receive_events_sqs_queue_name } diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf index 19ec453163..d06e68b736 100644 --- a/terraform/environment/region/variables.tf +++ b/terraform/environment/region/variables.tf @@ -112,12 +112,6 @@ variable "event_bus_enabled" { default = false } -variable "event_receiver_lambda_name" { - description = "The name of the event receiver lambda." - type = string - default = "" -} - variable "feature_flags" { description = "The feature flags to use." type = map(string) From d1f052d676ab5335b82b905769a3dc6fc644c1f1 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Thu, 19 Dec 2024 09:29:14 +0000 Subject: [PATCH 11/19] rewmove lambda var --- terraform/environment/region.tf | 2 -- terraform/environment/region/event_bus.tf | 11 +++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/terraform/environment/region.tf b/terraform/environment/region.tf index 94ae5d4f65..c2f8419aab 100644 --- a/terraform/environment/region.tf +++ b/terraform/environment/region.tf @@ -20,7 +20,6 @@ module "eu_west_1" { ecs_task_roles = module.iam.ecs_task_roles environment_name = local.environment_name event_bus_enabled = local.environment.event_bus_enabled - event_receiver_lambda_name = module.event_receiver.lambda_name google_analytics_id_use = local.environment.google_analytics_id_use google_analytics_id_view = local.environment.google_analytics_id_view gov_uk_onelogin_client_id_secret_name = local.environment.gov_uk_onelogin_client_id_secret_name @@ -109,7 +108,6 @@ module "eu_west_2" { ecs_task_roles = module.iam.ecs_task_roles environment_name = local.environment_name event_bus_enabled = local.environment.event_bus_enabled - event_receiver_lambda_name = module.event_receiver.lambda_name google_analytics_id_use = local.environment.google_analytics_id_use google_analytics_id_view = local.environment.google_analytics_id_view gov_uk_onelogin_client_id_secret_name = local.environment.gov_uk_onelogin_client_id_secret_name diff --git a/terraform/environment/region/event_bus.tf b/terraform/environment/region/event_bus.tf index 5ef9c8947a..f5d863c5e1 100644 --- a/terraform/environment/region/event_bus.tf +++ b/terraform/environment/region/event_bus.tf @@ -1,10 +1,9 @@ module "event_bus" { - source = "./modules/event_bus" - environment_name = var.environment_name - event_bus_enabled = var.event_bus_enabled - current_region = data.aws_region.current.name - receive_account_ids = var.receive_account_ids - event_receiver_lambda_name = var.event_receiver_lambda_name + source = "./modules/event_bus" + environment_name = var.environment_name + event_bus_enabled = var.event_bus_enabled + current_region = data.aws_region.current.name + receive_account_ids = var.receive_account_ids providers = { aws.region = aws.region } From ecad8bf90d86f0566ce41c6c42a9c8ac50a81002 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Thu, 19 Dec 2024 15:58:35 +0000 Subject: [PATCH 12/19] add lambda permission and change to arn --- terraform/environment/lambda.tf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index 0b0e756719..051afd1e7e 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -119,7 +119,7 @@ data "aws_iam_policy_document" "lambda_event_receiver" { "sqs:DeleteMessage", "sqs:GetQueueAttributes" ] - resources = [module.eu_west_1[0].receive_events_sqs_queue_name[0]] + resources = [module.eu_west_1[0].receive_events_sqs_queue_arn[0]] } } @@ -128,3 +128,11 @@ resource "aws_lambda_event_source_mapping" "receive_events_mapping" { function_name = module.event_receiver.lambda_name enabled = true } + +resource "aws_lambda_permission" "receive_events_permission" { + statement_id = "AllowExecutionFromSQS" + action = "lambda:InvokeFunction" + function_name = module.event_receiver.lambda_name + principal = "sqs.amazonaws.com" + source_arn = module.eu_west_1[0].receive_events_sqs_queue_arn[0] +} From 2387dbab061d96d81b63d66d512e6c2821c74959 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Thu, 19 Dec 2024 16:07:28 +0000 Subject: [PATCH 13/19] ensure queue visibility timeout aligns --- terraform/environment/region/event_bus.tf | 11 ++++++----- terraform/environment/region/locals.tf | 2 ++ terraform/environment/region/modules/event_bus/sqs.tf | 2 +- .../environment/region/modules/event_bus/variables.tf | 5 +++++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/terraform/environment/region/event_bus.tf b/terraform/environment/region/event_bus.tf index f5d863c5e1..701d4d1f28 100644 --- a/terraform/environment/region/event_bus.tf +++ b/terraform/environment/region/event_bus.tf @@ -1,9 +1,10 @@ module "event_bus" { - source = "./modules/event_bus" - environment_name = var.environment_name - event_bus_enabled = var.event_bus_enabled - current_region = data.aws_region.current.name - receive_account_ids = var.receive_account_ids + source = "./modules/event_bus" + environment_name = var.environment_name + event_bus_enabled = var.event_bus_enabled + current_region = data.aws_region.current.name + receive_account_ids = var.receive_account_ids + queue_visibility_timeout = local.queue_visibility_timeout providers = { aws.region = aws.region } diff --git a/terraform/environment/region/locals.tf b/terraform/environment/region/locals.tf index 6b5d6c5bdc..0dd9c5983c 100644 --- a/terraform/environment/region/locals.tf +++ b/terraform/environment/region/locals.tf @@ -14,6 +14,8 @@ locals { admin_desired_count = local.is_active_region ? 1 : 0 mock_onelogin_desired_count = var.environment_name != "production" && var.mock_onelogin_enabled && local.is_active_region ? 1 : 0 + queue_visibility_timeout = 900 + # Replace the region in the ARN of the DynamoDB tables with the region of the current stack as the tables are created in the primary region # and replicated to the secondary region. This allows use to grant access to the tables in the secondary region for applications running in the secondary region. dynamodb_tables_arns = { diff --git a/terraform/environment/region/modules/event_bus/sqs.tf b/terraform/environment/region/modules/event_bus/sqs.tf index 976e8eddef..8c0cb5ef68 100644 --- a/terraform/environment/region/modules/event_bus/sqs.tf +++ b/terraform/environment/region/modules/event_bus/sqs.tf @@ -9,7 +9,7 @@ resource "aws_sqs_queue" "receive_events_queue" { kms_master_key_id = data.aws_kms_alias.sqs.target_key_id kms_data_key_reuse_period_seconds = 300 - visibility_timeout_seconds = 300 + visibility_timeout_seconds = var.queue_visibility_timeout redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.receive_events_deadletter[0].arn diff --git a/terraform/environment/region/modules/event_bus/variables.tf b/terraform/environment/region/modules/event_bus/variables.tf index 1c22663c67..ca929547be 100644 --- a/terraform/environment/region/modules/event_bus/variables.tf +++ b/terraform/environment/region/modules/event_bus/variables.tf @@ -18,3 +18,8 @@ variable "receive_account_ids" { description = "The account ids that can send events to the event bus" type = list(string) } + +variable "queue_visibility_timeout" { + description = "The visibility timeout for the SQS queue" + type = number +} From dce71bc243600b86363848ca60c3d4c7e2768304 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Fri, 20 Dec 2024 09:17:16 +0000 Subject: [PATCH 14/19] correct ecr --- terraform/environment/lambda.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index 051afd1e7e..30d7484b94 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -14,6 +14,7 @@ module "lambda_update_statistics" { memory = 1024 } + # Additional IAM permissions resource "aws_iam_role_policy" "lambda_update_statistics" { name = "lambda-update-statistics-${local.environment_name}" @@ -96,7 +97,7 @@ module "event_receiver" { REGION = data.aws_region.current.name } image_uri = "${data.aws_ecr_repository.use_an_lpa_event_receiver.repository_url}:${var.container_version}" - ecr_arn = data.aws_ecr_repository.use_an_lpa_upload_statistics.arn + ecr_arn = data.aws_ecr_repository.use_an_lpa_event_receiver.arn environment = local.environment_name kms_key = data.aws_kms_alias.cloudwatch_encryption.target_key_arn timeout = 900 From 87cbdeb56396a79bee0685627177d8f1c27c33de Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Fri, 20 Dec 2024 14:11:23 +0000 Subject: [PATCH 15/19] stop pipeline pass on failed preprod plan --- .github/workflows/pull-request-path.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-path.yml b/.github/workflows/pull-request-path.yml index 1d1442b44d..0f0f78bd38 100644 --- a/.github/workflows/pull-request-path.yml +++ b/.github/workflows/pull-request-path.yml @@ -298,6 +298,7 @@ jobs: - update_documentation - docker_build_scan_push - run_behat_suite + - terraform_preproduction_plan_environment steps: - uses: actions/checkout@v4 @@ -332,7 +333,7 @@ jobs: - name: workflow has ended without issue run: | - if ${{ contains(needs.run_behat_suite.result, 'success') && contains(needs.ecr_scan_results.result, 'success') }}; then + if ${{ contains(needs.run_behat_suite.result, 'success') && contains(needs.ecr_scan_results.result, 'success') && contains(needs.terraform_preproduction_plan_environment.result, 'success') }}; then echo "${{ needs.workflow_variables.outputs.safe_branch_name }} PR environment tested, built and deployed" echo "Tag Used: ${{ needs.workflow_variables.outputs.safe_branch_name }}-${{ needs.workflow_variables.outputs.short_sha }}" echo "URL: https://${{ needs.workflow_variables.outputs.workspace_name }}.use-lasting-power-of-attorney.service.gov.uk" From 30642f4ad976f5d3c383bf60e837ba46f7c20ff7 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Fri, 20 Dec 2024 15:30:39 +0000 Subject: [PATCH 16/19] update lambda to allow execute from sqs --- lambda-functions/event-receiver/app/main.go | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lambda-functions/event-receiver/app/main.go b/lambda-functions/event-receiver/app/main.go index 7a74dacd67..8967bfcd49 100755 --- a/lambda-functions/event-receiver/app/main.go +++ b/lambda-functions/event-receiver/app/main.go @@ -1,17 +1,30 @@ package main import ( - "context" "fmt" + "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-lambda-go/lambda" ) -func Handler(ctx context.Context) (string, error) { - fmt.Println("Hello World") - return "Hello World!", nil +func handler(event events.SQSEvent) error { + for _, record := range event.Records { + err := processMessage(record) + if err != nil { + return err + } + } + fmt.Println("done") + return nil +} + +func processMessage(record events.SQSMessage) error { + fmt.Printf("Processed message %s\n", record.Body) + fmt.Printf("Hello, world!\n") + return nil } func main() { - lambda.Start(Handler) + lambda.Start(handler) + fmt.Printf("Hello, world!\n") } From 55f74886b37c2ffbc58d45e885c0a5bf42e61392 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Fri, 20 Dec 2024 15:57:51 +0000 Subject: [PATCH 17/19] explicitly declare resource --- terraform/environment/region/modules/event_bus/sqs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environment/region/modules/event_bus/sqs.tf b/terraform/environment/region/modules/event_bus/sqs.tf index 8c0cb5ef68..79e2113079 100644 --- a/terraform/environment/region/modules/event_bus/sqs.tf +++ b/terraform/environment/region/modules/event_bus/sqs.tf @@ -52,7 +52,7 @@ data "aws_iam_policy_document" "receive_events_queue_policy" { } actions = ["sqs:SendMessage"] - resources = ["*"] + resources = [aws_sqs_queue.receive_events_queue[0].arn] condition { test = "ArnEquals" From f8369133d54b7537bbb08f035c461626995ce33d Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Fri, 20 Dec 2024 16:15:43 +0000 Subject: [PATCH 18/19] add sqs queue policy resource --- terraform/environment/region/modules/event_bus/sqs.tf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/terraform/environment/region/modules/event_bus/sqs.tf b/terraform/environment/region/modules/event_bus/sqs.tf index 79e2113079..25efd3e0c5 100644 --- a/terraform/environment/region/modules/event_bus/sqs.tf +++ b/terraform/environment/region/modules/event_bus/sqs.tf @@ -16,7 +16,13 @@ resource "aws_sqs_queue" "receive_events_queue" { maxReceiveCount = 3 }) - policy = data.aws_iam_policy_document.receive_events_queue_policy[0].json + provider = aws.region +} + +resource "aws_sqs_queue_policy" "receive_events_queue_policy" { + count = var.event_bus_enabled ? 1 : 0 + queue_url = aws_sqs_queue.receive_events_queue[0].id + policy = data.aws_iam_policy_document.receive_events_queue_policy[0].json provider = aws.region } From 5921e22d0118b2ecb0faa79db2b2301f8421b495 Mon Sep 17 00:00:00 2001 From: Jay Whitwell Date: Fri, 20 Dec 2024 16:38:30 +0000 Subject: [PATCH 19/19] allow lambda decrypt permissions --- terraform/environment/lambda.tf | 9 +++++++++ terraform/environment/shared_data_sources.tf | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/terraform/environment/lambda.tf b/terraform/environment/lambda.tf index 30d7484b94..5e3534fbb5 100644 --- a/terraform/environment/lambda.tf +++ b/terraform/environment/lambda.tf @@ -122,6 +122,15 @@ data "aws_iam_policy_document" "lambda_event_receiver" { ] resources = [module.eu_west_1[0].receive_events_sqs_queue_arn[0]] } + + statement { + sid = "${local.environment_name}SQSKMSDecrypt" + effect = "Allow" + actions = [ + "kms:Decrypt" + ] + resources = [data.aws_kms_alias.sqs.arn] + } } resource "aws_lambda_event_source_mapping" "receive_events_mapping" { diff --git a/terraform/environment/shared_data_sources.tf b/terraform/environment/shared_data_sources.tf index c55ab490df..35d2f567f6 100644 --- a/terraform/environment/shared_data_sources.tf +++ b/terraform/environment/shared_data_sources.tf @@ -20,6 +20,10 @@ data "aws_ecr_repository" "mock_onelogin" { name = "mock-onelogin" } +data "aws_kms_alias" "sqs" { + name = "alias/sqs-mrk" +} + module "allow_list" { source = "git@github.com:ministryofjustice/terraform-aws-moj-ip-allow-list.git?ref=v2.3.0" }