From e617e18f8f0f9a082b8d3d8811d2bacc5de619f4 Mon Sep 17 00:00:00 2001
From: Sam Ainsworth <samuel.ainsworth@digital.justice.gov.uk>
Date: Mon, 16 Oct 2023 10:28:13 +0100
Subject: [PATCH] UML-3114 Use global DynamoDB tables (#2365)

* UML-3114 Use global DynamoDB tables
---
 terraform/environment/dynamodb.tf           | 135 +++++++++++++++++---
 terraform/environment/locals.tf             |   9 +-
 terraform/environment/region.tf             |   2 +-
 terraform/environment/region/actor_ecs.tf   |   2 +-
 terraform/environment/region/admin_ecs.tf   |  22 ++--
 terraform/environment/region/api_ecs.tf     |  26 ++--
 terraform/environment/region/locals.tf      |  25 +++-
 terraform/environment/region/pdf_ecs.tf     |   2 +-
 terraform/environment/region/variables.tf   |  13 ++
 terraform/environment/region/viewer_ecs.tf  |   2 +-
 terraform/environment/terraform.tfvars.json |  33 +++++
 11 files changed, 222 insertions(+), 49 deletions(-)

diff --git a/terraform/environment/dynamodb.tf b/terraform/environment/dynamodb.tf
index c24e33cce2..1af40e444b 100644
--- a/terraform/environment/dynamodb.tf
+++ b/terraform/environment/dynamodb.tf
@@ -1,7 +1,9 @@
 resource "aws_dynamodb_table" "actor_codes_table" {
-  name         = "${local.environment_name}-${local.environment.dynamodb_tables.actor_codes.name}"
-  billing_mode = "PAY_PER_REQUEST"
-  hash_key     = "ActorCode"
+  name             = "${local.environment_name}-${local.environment.dynamodb_tables.actor_codes.name}"
+  billing_mode     = "PAY_PER_REQUEST"
+  hash_key         = "ActorCode"
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
   server_side_encryption {
     enabled = true
   }
@@ -15,15 +17,34 @@ resource "aws_dynamodb_table" "actor_codes_table" {
     enabled = true
   }
 
+  # For each region in the environment that is not the primary_region, create a DynamoDB replica.
+
+  dynamic "replica" {
+    for_each = [
+      for region in local.environment.regions : region
+      if region.is_primary != true
+    ]
+
+    content {
+      region_name    = replica.value.name
+      propagate_tags = true
+
+    }
+  }
+
   lifecycle {
     prevent_destroy = false
   }
+
+  provider = aws.eu_west_1
 }
 
 resource "aws_dynamodb_table" "stats_table" {
-  name         = "${local.environment_name}-${local.environment.dynamodb_tables.stats.name}"
-  billing_mode = "PAY_PER_REQUEST"
-  hash_key     = "TimePeriod"
+  name             = "${local.environment_name}-${local.environment.dynamodb_tables.stats.name}"
+  billing_mode     = "PAY_PER_REQUEST"
+  hash_key         = "TimePeriod"
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
   #tfsec:ignore:aws-dynamodb-table-customer-key - same as the other tables. Will update in one go as separate ticket
   server_side_encryption {
     enabled = true
@@ -38,15 +59,31 @@ resource "aws_dynamodb_table" "stats_table" {
     enabled = true
   }
 
+  dynamic "replica" {
+    for_each = [
+      for region in local.environment.regions : region
+      if region.is_primary != true
+    ]
+
+    content {
+      region_name    = replica.value.name
+      propagate_tags = true
+    }
+  }
+
   lifecycle {
     prevent_destroy = false
   }
+
+  provider = aws.eu_west_1
 }
 
 resource "aws_dynamodb_table" "actor_users_table" {
-  name         = "${local.environment_name}-${local.environment.dynamodb_tables.actor_users.name}"
-  billing_mode = "PAY_PER_REQUEST"
-  hash_key     = "Id"
+  name             = "${local.environment_name}-${local.environment.dynamodb_tables.actor_users.name}"
+  billing_mode     = "PAY_PER_REQUEST"
+  hash_key         = "Id"
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
   server_side_encryption {
     enabled = true
   }
@@ -111,16 +148,31 @@ resource "aws_dynamodb_table" "actor_users_table" {
     enabled = true
   }
 
+  dynamic "replica" {
+    for_each = [
+      for region in local.environment.regions : region
+      if region.is_primary != true
+    ]
+
+    content {
+      region_name    = replica.value.name
+      propagate_tags = true
+    }
+  }
 
   lifecycle {
     prevent_destroy = false
   }
+
+  provider = aws.eu_west_1
 }
 
 resource "aws_dynamodb_table" "viewer_codes_table" {
-  name         = "${local.environment_name}-${local.environment.dynamodb_tables.viewer_codes.name}"
-  billing_mode = "PAY_PER_REQUEST"
-  hash_key     = "ViewerCode"
+  name             = "${local.environment_name}-${local.environment.dynamodb_tables.viewer_codes.name}"
+  billing_mode     = "PAY_PER_REQUEST"
+  hash_key         = "ViewerCode"
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
   server_side_encryption {
     enabled = true
   }
@@ -151,17 +203,33 @@ resource "aws_dynamodb_table" "viewer_codes_table" {
     enabled = true
   }
 
+  dynamic "replica" {
+    for_each = [
+      for region in local.environment.regions : region
+      if region.is_primary != true
+    ]
+
+    content {
+      region_name    = replica.value.name
+      propagate_tags = true
+    }
+  }
+
 
   lifecycle {
     prevent_destroy = false
   }
+
+  provider = aws.eu_west_1
 }
 
 resource "aws_dynamodb_table" "viewer_activity_table" {
-  name         = "${local.environment_name}-${local.environment.dynamodb_tables.viewer_activity.name}"
-  billing_mode = "PAY_PER_REQUEST"
-  hash_key     = "ViewerCode"
-  range_key    = "Viewed"
+  name             = "${local.environment_name}-${local.environment.dynamodb_tables.viewer_activity.name}"
+  billing_mode     = "PAY_PER_REQUEST"
+  hash_key         = "ViewerCode"
+  range_key        = "Viewed"
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
   server_side_encryption {
     enabled = true
   }
@@ -179,16 +247,32 @@ resource "aws_dynamodb_table" "viewer_activity_table" {
     enabled = true
   }
 
+  dynamic "replica" {
+    for_each = [
+      for region in local.environment.regions : region
+      if region.is_primary != true
+    ]
+
+    content {
+      region_name    = replica.value.name
+      propagate_tags = true
+    }
+  }
+
 
   lifecycle {
     prevent_destroy = false
   }
+
+  provider = aws.eu_west_1
 }
 
 resource "aws_dynamodb_table" "user_lpa_actor_map" {
-  name         = "${local.environment_name}-${local.environment.dynamodb_tables.user_lpa_actor_map.name}"
-  billing_mode = "PAY_PER_REQUEST"
-  hash_key     = "Id"
+  name             = "${local.environment_name}-${local.environment.dynamodb_tables.user_lpa_actor_map.name}"
+  billing_mode     = "PAY_PER_REQUEST"
+  hash_key         = "Id"
+  stream_enabled   = true
+  stream_view_type = "NEW_AND_OLD_IMAGES"
   server_side_encryption {
     enabled = true
   }
@@ -240,8 +324,21 @@ resource "aws_dynamodb_table" "user_lpa_actor_map" {
     enabled = true
   }
 
+  dynamic "replica" {
+    for_each = [
+      for region in local.environment.regions : region
+      if region.is_primary != true
+    ]
+
+    content {
+      region_name    = replica.value.name
+      propagate_tags = true
+    }
+  }
 
   lifecycle {
     prevent_destroy = false
   }
+
+  provider = aws.eu_west_1
 }
diff --git a/terraform/environment/locals.tf b/terraform/environment/locals.tf
index 17ad3e0399..a5cbf801be 100644
--- a/terraform/environment/locals.tf
+++ b/terraform/environment/locals.tf
@@ -104,7 +104,14 @@ variable "environments" {
         stats = object({
           name = string
         })
-      })
+      }),
+      regions = map(
+        object({
+          name       = string
+          is_active  = bool
+          is_primary = bool
+        })
+      )
     })
   )
 }
diff --git a/terraform/environment/region.tf b/terraform/environment/region.tf
index 39e3cdf9df..fe1fe38e5e 100644
--- a/terraform/environment/region.tf
+++ b/terraform/environment/region.tf
@@ -25,6 +25,7 @@ module "eu_west_1" {
   parameter_store_arns                      = [aws_ssm_parameter.system_message_view_en.arn, aws_ssm_parameter.system_message_view_cy.arn, aws_ssm_parameter.system_message_use_en.arn, aws_ssm_parameter.system_message_use_cy.arn]
   pdf_container_version                     = local.environment.pdf_container_version
   public_access_enabled                     = var.public_access_enabled
+  regions                                   = local.environment.regions
   session_expires_use                       = local.environment.session_expires_use
   session_expires_view                      = local.environment.session_expires_view
   session_expiry_warning                    = local.environment.session_expiry_warning
@@ -73,7 +74,6 @@ module "eu_west_1" {
     "viewer"      = aws_route53_record.viewer_use_my_lpa.fqdn
   }
 
-
   providers = {
     aws.region     = aws.eu_west_1
     aws.management = aws.management
diff --git a/terraform/environment/region/actor_ecs.tf b/terraform/environment/region/actor_ecs.tf
index 51e6e2a24b..17d1776acb 100644
--- a/terraform/environment/region/actor_ecs.tf
+++ b/terraform/environment/region/actor_ecs.tf
@@ -5,7 +5,7 @@ resource "aws_ecs_service" "actor" {
   name             = "actor-service"
   cluster          = aws_ecs_cluster.use_an_lpa.id
   task_definition  = aws_ecs_task_definition.actor.arn
-  desired_count    = var.autoscaling.use.minimum
+  desired_count    = local.use_desired_count
   platform_version = "1.4.0"
 
   network_configuration {
diff --git a/terraform/environment/region/admin_ecs.tf b/terraform/environment/region/admin_ecs.tf
index 17f5e0aabd..ab7f99f6b2 100644
--- a/terraform/environment/region/admin_ecs.tf
+++ b/terraform/environment/region/admin_ecs.tf
@@ -5,7 +5,7 @@ resource "aws_ecs_service" "admin" {
   name             = "admin-service"
   cluster          = aws_ecs_cluster.use_an_lpa.id
   task_definition  = aws_ecs_task_definition.admin.arn
-  desired_count    = 1
+  desired_count    = local.admin_desired_count
   platform_version = "1.4.0"
 
   network_configuration {
@@ -165,16 +165,16 @@ data "aws_iam_policy_document" "admin_permissions_role" {
     ]
 
     resources = [
-      var.dynamodb_tables.actor_users_table.arn,
-      "${var.dynamodb_tables.actor_users_table.arn}/index/*",
-      var.dynamodb_tables.viewer_codes_table.arn,
-      "${var.dynamodb_tables.viewer_codes_table.arn}/index/*",
-      var.dynamodb_tables.viewer_activity_table.arn,
-      "${var.dynamodb_tables.viewer_activity_table.arn}/index/*",
-      var.dynamodb_tables.user_lpa_actor_map.arn,
-      "${var.dynamodb_tables.user_lpa_actor_map.arn}/index/*",
-      var.dynamodb_tables.stats_table.arn,
-      "${var.dynamodb_tables.stats_table.arn}/index/*",
+      local.dynamodb_tables_arns.actor_users_table_arn,
+      "${local.dynamodb_tables_arns.actor_users_table_arn}/index/*",
+      local.dynamodb_tables_arns.viewer_codes_table_arn,
+      "${local.dynamodb_tables_arns.viewer_codes_table_arn}/index/*",
+      local.dynamodb_tables_arns.viewer_activity_table_arn,
+      "${local.dynamodb_tables_arns.viewer_activity_table_arn}/index/*",
+      local.dynamodb_tables_arns.user_lpa_actor_map_arn,
+      "${local.dynamodb_tables_arns.user_lpa_actor_map_arn}/index/*",
+      local.dynamodb_tables_arns.stats_table_arn,
+      "${local.dynamodb_tables_arns.stats_table_arn}/index/*",
     ]
   }
 
diff --git a/terraform/environment/region/api_ecs.tf b/terraform/environment/region/api_ecs.tf
index 5823a957a6..4effea21f5 100644
--- a/terraform/environment/region/api_ecs.tf
+++ b/terraform/environment/region/api_ecs.tf
@@ -5,7 +5,7 @@ resource "aws_ecs_service" "api" {
   name                              = "api-service"
   cluster                           = aws_ecs_cluster.use_an_lpa.id
   task_definition                   = aws_ecs_task_definition.api.arn
-  desired_count                     = var.autoscaling.api.minimum
+  desired_count                     = local.api_desired_count
   platform_version                  = "1.4.0"
   health_check_grace_period_seconds = 0
 
@@ -193,18 +193,18 @@ data "aws_iam_policy_document" "api_permissions_role" {
     ]
 
     resources = [
-      var.dynamodb_tables.actor_codes_table.arn,
-      "${var.dynamodb_tables.actor_codes_table.arn}/index/*",
-      var.dynamodb_tables.actor_users_table.arn,
-      "${var.dynamodb_tables.actor_users_table.arn}/index/*",
-      var.dynamodb_tables.viewer_codes_table.arn,
-      "${var.dynamodb_tables.viewer_codes_table.arn}/index/*",
-      var.dynamodb_tables.viewer_activity_table.arn,
-      "${var.dynamodb_tables.viewer_activity_table.arn}/index/*",
-      var.dynamodb_tables.user_lpa_actor_map.arn,
-      "${var.dynamodb_tables.user_lpa_actor_map.arn}/index/*",
-      var.dynamodb_tables.stats_table.arn,
-      "${var.dynamodb_tables.stats_table.arn}/index/*",
+      local.dynamodb_tables_arns.actor_codes_table_arn,
+      "${local.dynamodb_tables_arns.actor_codes_table_arn}/index/*",
+      local.dynamodb_tables_arns.actor_users_table_arn,
+      "${local.dynamodb_tables_arns.actor_users_table_arn}/index/*",
+      local.dynamodb_tables_arns.viewer_codes_table_arn,
+      "${local.dynamodb_tables_arns.viewer_codes_table_arn}/index/*",
+      local.dynamodb_tables_arns.viewer_activity_table_arn,
+      "${local.dynamodb_tables_arns.viewer_activity_table_arn}/index/*",
+      local.dynamodb_tables_arns.user_lpa_actor_map_arn,
+      "${local.dynamodb_tables_arns.user_lpa_actor_map_arn}/index/*",
+      local.dynamodb_tables_arns.stats_table_arn,
+      "${local.dynamodb_tables_arns.stats_table_arn}/index/*",
     ]
   }
 
diff --git a/terraform/environment/region/locals.tf b/terraform/environment/region/locals.tf
index ecf2610f5c..ca9c6f5655 100644
--- a/terraform/environment/region/locals.tf
+++ b/terraform/environment/region/locals.tf
@@ -1,3 +1,26 @@
 locals {
   policy_region_prefix = lower(replace(data.aws_region.current.name, "-", ""))
-}
\ No newline at end of file
+
+  # The primary region is the region where the DynamoDB tables are created and replicated to the secondary region.
+  primary_region    = keys({ for region, region_data in var.regions : region => region_data if region_data.is_primary })[0]
+  is_primary_region = local.primary_region == data.aws_region.current.name ? true : false
+  is_active_region  = var.regions[data.aws_region.current.name].is_active
+
+  # Desired count of the ECS services. Only an active region will have a desired count greater than 0.
+  use_desired_count   = local.is_active_region ? var.autoscaling.use.minimum : 0
+  pdf_desired_count   = local.is_active_region ? var.autoscaling.pdf.minimum : 0
+  view_desired_count  = local.is_active_region ? var.autoscaling.view.minimum : 0
+  api_desired_count   = local.is_active_region ? var.autoscaling.api.minimum : 0
+  admin_desired_count = local.is_active_region ? 1 : 0
+
+  # Replace the region in the ARN of the DynamoDB tables with the region of the current stack as the tables are created in the primary region
+  # and replicated to the secondary region. This allows use to grant access to the tables in the secondary region for applications running in the secondary region.
+  dynamodb_tables_arns = {
+    actor_codes_table_arn     = replace(var.dynamodb_tables.actor_codes_table.arn, local.primary_region, data.aws_region.current.name)
+    stats_table_arn           = replace(var.dynamodb_tables.stats_table.arn, local.primary_region, data.aws_region.current.name)
+    actor_users_table_arn     = replace(var.dynamodb_tables.actor_users_table.arn, local.primary_region, data.aws_region.current.name)
+    viewer_codes_table_arn    = replace(var.dynamodb_tables.viewer_codes_table.arn, local.primary_region, data.aws_region.current.name)
+    viewer_activity_table_arn = replace(var.dynamodb_tables.viewer_activity_table.arn, local.primary_region, data.aws_region.current.name)
+    user_lpa_actor_map_arn    = replace(var.dynamodb_tables.user_lpa_actor_map.arn, local.primary_region, data.aws_region.current.name)
+  }
+}
diff --git a/terraform/environment/region/pdf_ecs.tf b/terraform/environment/region/pdf_ecs.tf
index 818a5573c3..ac93f08f70 100644
--- a/terraform/environment/region/pdf_ecs.tf
+++ b/terraform/environment/region/pdf_ecs.tf
@@ -5,7 +5,7 @@ resource "aws_ecs_service" "pdf" {
   name             = "pdf-service"
   cluster          = aws_ecs_cluster.use_an_lpa.id
   task_definition  = aws_ecs_task_definition.pdf.arn
-  desired_count    = var.autoscaling.pdf.minimum
+  desired_count    = local.pdf_desired_count
   platform_version = "1.4.0"
 
   network_configuration {
diff --git a/terraform/environment/region/variables.tf b/terraform/environment/region/variables.tf
index 31ac77b91d..bb6255ec26 100644
--- a/terraform/environment/region/variables.tf
+++ b/terraform/environment/region/variables.tf
@@ -174,6 +174,19 @@ variable "public_access_enabled" {
   default     = false
 }
 
+variable "regions" {
+  description = "Information about which regions are being used"
+  type = map(object({
+    is_primary = bool
+    is_active  = bool
+  }))
+
+  validation {
+    condition     = length([for region in keys(var.regions) : region if var.regions[region].is_primary]) == 1
+    error_message = "One (and only one) region must be marked as primary"
+  }
+}
+
 variable "route_53_fqdns" {
   description = "The FQDNs to use for the Route 53 records."
 
diff --git a/terraform/environment/region/viewer_ecs.tf b/terraform/environment/region/viewer_ecs.tf
index 1e730e0b11..a95b5f4b66 100644
--- a/terraform/environment/region/viewer_ecs.tf
+++ b/terraform/environment/region/viewer_ecs.tf
@@ -5,7 +5,7 @@ resource "aws_ecs_service" "viewer" {
   name             = "viewer-service"
   cluster          = aws_ecs_cluster.use_an_lpa.id
   task_definition  = aws_ecs_task_definition.viewer.arn
-  desired_count    = var.autoscaling.view.minimum
+  desired_count    = local.view_desired_count
   platform_version = "1.4.0"
 
   network_configuration {
diff --git a/terraform/environment/terraform.tfvars.json b/terraform/environment/terraform.tfvars.json
index a4aab5d701..9b09d8ce3e 100644
--- a/terraform/environment/terraform.tfvars.json
+++ b/terraform/environment/terraform.tfvars.json
@@ -73,6 +73,13 @@
         "stats": {
           "name": "Stats"
         }
+      },
+      "regions": {
+        "eu-west-1": {
+          "name": "eu-west-1",
+          "is_active": true,
+          "is_primary": true
+        }
       }
     },
     "demo": {
@@ -148,6 +155,13 @@
         "stats": {
           "name": "Stats"
         }
+      },
+      "regions": {
+        "eu-west-1": {
+          "name": "eu-west-1",
+          "is_active": true,
+          "is_primary": true
+        }
       }
     },
     "preproduction": {
@@ -223,6 +237,18 @@
         "stats": {
           "name": "Stats"
         }
+      },
+      "regions": {
+        "eu-west-1": {
+          "name": "eu-west-1",
+          "is_active": true,
+          "is_primary": true
+        },
+        "eu-west-2": {
+          "name": "eu-west-2",
+          "is_active": true,
+          "is_primary": false
+        }
       }
     },
     "production": {
@@ -298,6 +324,13 @@
         "stats": {
           "name": "Stats"
         }
+      },
+      "regions": {
+        "eu-west-1": {
+          "name": "eu-west-1",
+          "is_active": true,
+          "is_primary": true
+        }
       }
     }
   }