feat: refactor table scan with gsi query (#608)

* feat: replace table scan with query on GSI

* feat: add dynamodb table idpStatus configuration and update lambda permissions

* terraform-docs: automated action

* feat: add DynamoDB VPC endpoint prefix ID to backend modules

* terraform-docs: automated action

* feat: update entity_id list in uat env

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

src/infra/dev/eu-south-1/main.tf

@@ -298,6 +298,11 @@ module "backend" {
     table_arn       = module.database.table_idp_metadata_arn
   }
 
+  dynamodb_table_idpStatus = {
+    gsi_pointer_arn = module.database.table_idp_status_gsi_pointer_arn
+    table_arn       = module.database.table_idp_status_history_arn
+  }
+
   is_gh_integration_lambda = {
     name                              = format("%s-is-gh-integration-lambda", local.project)
     filename                          = "${path.module}/../../hello-java/build/libs/hello-java-1.0-SNAPSHOT.jar"
@@ -310,14 +315,15 @@ module "backend" {
     name                              = format("%s-update-idp-status", local.project)
     filename                          = "${path.module}/../../hello-python/lambda.zip"
     assets_bucket_arn                 = module.storage.assets_bucket_arn
-    table_idp_status_history_arn      = module.database.table_idp_status_history_arn
     vpc_id                            = module.network.vpc_id
     vpc_subnet_ids                    = module.network.intra_subnets_ids
     vpc_s3_prefix_id                  = module.network.vpc_endpoints["s3"]["prefix_list_id"]
+    vpc_endpoint_dynamodb_prefix_id   = module.network.vpc_endpoints["dynamodb"]["prefix_list_id"]
     cloudwatch_logs_retention_in_days = var.lambda_cloudwatch_logs_retention_in_days
     environment_variables = {
       LOG_LEVEL                 = var.app_log_level
       IDP_STATUS_DYNAMODB_TABLE = module.database.table_idp_status_history_name
+      IDP_STATUS_DYNAMODB_IDX   = module.database.table_idp_status_history_idx_name
       ASSETS_S3_BUCKET          = module.storage.assets_bucket_name
       IDP_STATUS_S3_FILE_NAME   = "idp_status_history.json"
     }

src/infra/modules/backend/README.md

@@ -214,6 +214,7 @@
 | <a name="input_dlq_alarms"></a> [dlq\_alarms](#input\_dlq\_alarms) | n/a | <pre>object({<br>    metric_name         = string<br>    namespace           = string<br>    threshold           = number<br>    evaluation_periods  = number<br>    period              = number<br>    statistic           = string<br>    comparison_operator = string<br>    sns_topic_alarm_arn = string<br>  })</pre> | n/a | yes |
 | <a name="input_dynamodb_clients_table_stream_arn"></a> [dynamodb\_clients\_table\_stream\_arn](#input\_dynamodb\_clients\_table\_stream\_arn) | n/a | `string` | `null` | no |
 | <a name="input_dynamodb_table_idpMetadata"></a> [dynamodb\_table\_idpMetadata](#input\_dynamodb\_table\_idpMetadata) | Dynamodb table idpMetadata anrs | <pre>object({<br>    table_arn       = string<br>    gsi_pointer_arn = string<br>  })</pre> | n/a | yes |
+| <a name="input_dynamodb_table_idpStatus"></a> [dynamodb\_table\_idpStatus](#input\_dynamodb\_table\_idpStatus) | Dynamodb table idpStatus arns | <pre>object({<br>    table_arn       = string<br>    gsi_pointer_arn = string<br>  })</pre> | n/a | yes |
 | <a name="input_dynamodb_table_sessions"></a> [dynamodb\_table\_sessions](#input\_dynamodb\_table\_sessions) | Dynamodb table sessions anrs | <pre>object({<br>    table_arn    = string<br>    gsi_code_arn = string<br>  })</pre> | n/a | yes |
 | <a name="input_dynamodb_table_stream_arn"></a> [dynamodb\_table\_stream\_arn](#input\_dynamodb\_table\_stream\_arn) | n/a | `string` | `null` | no |
 | <a name="input_ecr_registers"></a> [ecr\_registers](#input\_ecr\_registers) | ECR image repositories | <pre>list(object({<br>    name                            = string<br>    number_of_images_to_keep        = number<br>    repository_image_tag_mutability = optional(string, "IMMUTABLE")<br>  }))</pre> | n/a | yes |
@@ -241,7 +242,7 @@
 | <a name="input_ssm_cert_key"></a> [ssm\_cert\_key](#input\_ssm\_cert\_key) | TODO fix name | <pre>object({<br>    cert_pem = optional(string, "cert.pem")<br>    key_pem  = optional(string, "key.pem")<br>  })</pre> | n/a | yes |
 | <a name="input_switch_region_enabled"></a> [switch\_region\_enabled](#input\_switch\_region\_enabled) | n/a | `bool` | `false` | no |
 | <a name="input_table_client_registrations_arn"></a> [table\_client\_registrations\_arn](#input\_table\_client\_registrations\_arn) | Dynamodb table client registrations arn. | `string` | n/a | yes |
-| <a name="input_update_idp_status_lambda"></a> [update\_idp\_status\_lambda](#input\_update\_idp\_status\_lambda) | n/a | <pre>object({<br>    name                              = string<br>    filename                          = string<br>    assets_bucket_arn                 = string<br>    table_idp_status_history_arn      = string<br>    cloudwatch_logs_retention_in_days = string<br>    environment_variables             = map(string)<br>    vpc_s3_prefix_id                  = string<br>    vpc_subnet_ids                    = list(string)<br>    vpc_id                            = string<br>  })</pre> | n/a | yes |
+| <a name="input_update_idp_status_lambda"></a> [update\_idp\_status\_lambda](#input\_update\_idp\_status\_lambda) | n/a | <pre>object({<br>    name                              = string<br>    filename                          = string<br>    assets_bucket_arn                 = string<br>    cloudwatch_logs_retention_in_days = string<br>    environment_variables             = map(string)<br>    vpc_s3_prefix_id                  = string<br>    vpc_endpoint_dynamodb_prefix_id   = string<br>    vpc_subnet_ids                    = list(string)<br>    vpc_id                            = string<br>  })</pre> | n/a | yes |
 | <a name="input_vpc_cidr_block"></a> [vpc\_cidr\_block](#input\_vpc\_cidr\_block) | VPC cidr block. | `string` | n/a | yes |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | VPC id | `string` | n/a | yes |
 

src/infra/modules/backend/lambda.tf

@@ -598,7 +598,9 @@ data "aws_iam_policy_document" "update_idp_status_lambda" {
       "dynamodb:DeleteItem",
       "dynamodb:Query",
     "dynamodb:PutItem"]
-    resources = ["${var.update_idp_status_lambda.table_idp_status_history_arn}"]
+    resources = [
+      var.dynamodb_table_idpStatus.table_arn,
+    var.dynamodb_table_idpStatus.gsi_pointer_arn]
   }
 }
 
@@ -617,6 +619,7 @@ module "security_group_update_idp_status_lambda" {
 
   # Prefix list ids to use in all egress rules in this module
   egress_prefix_list_ids = [
+    var.update_idp_status_lambda.vpc_endpoint_dynamodb_prefix_id,
     var.update_idp_status_lambda.vpc_s3_prefix_id,
   ]
   egress_rules = ["https-443-tcp"]

src/infra/modules/backend/variables.tf

@@ -116,6 +116,15 @@ variable "dynamodb_table_idpMetadata" {
   description = "Dynamodb table idpMetadata anrs"
 }
 
+variable "dynamodb_table_idpStatus" {
+  type = object({
+    table_arn       = string
+    gsi_pointer_arn = string
+  })
+  description = "Dynamodb table idpStatus arns"
+}
+
+
 variable "table_client_registrations_arn" {
   type        = string
   description = "Dynamodb table client registrations arn."
@@ -250,10 +259,10 @@ variable "update_idp_status_lambda" {
     name                              = string
     filename                          = string
     assets_bucket_arn                 = string
-    table_idp_status_history_arn      = string
     cloudwatch_logs_retention_in_days = string
     environment_variables             = map(string)
     vpc_s3_prefix_id                  = string
+    vpc_endpoint_dynamodb_prefix_id   = string
     vpc_subnet_ids                    = list(string)
     vpc_id                            = string
   })

src/infra/modules/database/README.md

@@ -97,7 +97,9 @@ No resources.
 | <a name="output_table_idp_metadata_arn"></a> [table\_idp\_metadata\_arn](#output\_table\_idp\_metadata\_arn) | n/a |
 | <a name="output_table_idp_metadata_idx_name"></a> [table\_idp\_metadata\_idx\_name](#output\_table\_idp\_metadata\_idx\_name) | n/a |
 | <a name="output_table_idp_metadata_name"></a> [table\_idp\_metadata\_name](#output\_table\_idp\_metadata\_name) | n/a |
+| <a name="output_table_idp_status_gsi_pointer_arn"></a> [table\_idp\_status\_gsi\_pointer\_arn](#output\_table\_idp\_status\_gsi\_pointer\_arn) | n/a |
 | <a name="output_table_idp_status_history_arn"></a> [table\_idp\_status\_history\_arn](#output\_table\_idp\_status\_history\_arn) | n/a |
+| <a name="output_table_idp_status_history_idx_name"></a> [table\_idp\_status\_history\_idx\_name](#output\_table\_idp\_status\_history\_idx\_name) | n/a |
 | <a name="output_table_idp_status_history_name"></a> [table\_idp\_status\_history\_name](#output\_table\_idp\_status\_history\_name) | n/a |
 | <a name="output_table_sessions_arn"></a> [table\_sessions\_arn](#output\_table\_sessions\_arn) | n/a |
 | <a name="output_table_sessions_gsi_code_arn"></a> [table\_sessions\_gsi\_code\_arn](#output\_table\_sessions\_gsi\_code\_arn) | n/a |

src/infra/modules/database/outputs.tf

@@ -33,6 +33,10 @@ output "table_idp_status_history_arn" {
   value = try(module.dynamodb_table_idp_status_history[0].dynamodb_table_arn, null)
 }
 
+output "table_idp_status_gsi_pointer_arn" {
+  value = try("${module.dynamodb_table_idp_status_history[0].dynamodb_table_arn}/index/${local.gsi_pointer}", null)
+}
+
 
 output "kms_sessions_table_alias_arn" {
   value = module.kms_sessions_table.aliases[local.kms_sessions_table_alias].target_key_arn
@@ -57,6 +61,10 @@ output "table_idp_metadata_idx_name" {
   value = local.gsi_pointer
 }
 
+output "table_idp_status_history_idx_name" {
+  value = local.gsi_pointer
+}
+
 output "table_idp_metadata_arn" {
   value = try(module.dynamodb_table_idpMetadata[0].dynamodb_table_arn, null)
 }

src/infra/prod/eu-central-1/main.tf

@@ -269,6 +269,11 @@ module "backend" {
     table_arn       = local.table_idp_metadata_arn
   }
 
+  dynamodb_table_idpStatus = {
+    gsi_pointer_arn = module.database.table_idp_status_gsi_pointer_arn
+    table_arn       = module.database.table_idp_status_history_arn
+  }
+
   is_gh_integration_lambda = {
     name                              = format("%s-is-gh-integration-lambda", local.project)
     filename                          = "${path.module}/../../hello-java/build/libs/hello-java-1.0-SNAPSHOT.jar"
@@ -281,14 +286,15 @@ module "backend" {
     name                              = format("%s-update-idp-status", local.project)
     filename                          = "${path.module}/../../hello-python/lambda.zip"
     assets_bucket_arn                 = module.storage.assets_bucket_arn
-    table_idp_status_history_arn      = module.database.table_idp_status_history_arn
     vpc_id                            = module.network.vpc_id
     vpc_subnet_ids                    = module.network.intra_subnets_ids
     vpc_s3_prefix_id                  = module.network.vpc_endpoints["s3"]["prefix_list_id"]
+    vpc_endpoint_dynamodb_prefix_id   = module.network.vpc_endpoints["dynamodb"]["prefix_list_id"]
     cloudwatch_logs_retention_in_days = var.lambda_cloudwatch_logs_retention_in_days
     environment_variables = {
       LOG_LEVEL                 = var.app_log_level
       IDP_STATUS_DYNAMODB_TABLE = module.database.table_idp_status_history_name
+      IDP_STATUS_DYNAMODB_IDX   = module.database.table_idp_status_history_idx_name
       ASSETS_S3_BUCKET          = module.storage.assets_bucket_name
       IDP_STATUS_S3_FILE_NAME   = "idp_status_history.json"
     }

src/infra/prod/eu-south-1/main.tf

@@ -313,6 +313,11 @@ module "backend" {
     table_arn       = module.database.table_idp_metadata_arn
   }
 
+  dynamodb_table_idpStatus = {
+    gsi_pointer_arn = module.database.table_idp_status_gsi_pointer_arn
+    table_arn       = module.database.table_idp_status_history_arn
+  }
+
   is_gh_integration_lambda = {
     name                              = format("%s-is-gh-integration-lambda", local.project)
     filename                          = "${path.module}/../../hello-java/build/libs/hello-java-1.0-SNAPSHOT.jar"
@@ -325,14 +330,15 @@ module "backend" {
     name                              = format("%s-update-idp-status", local.project)
     filename                          = "${path.module}/../../hello-python/lambda.zip"
     assets_bucket_arn                 = module.storage.assets_bucket_arn
-    table_idp_status_history_arn      = module.database.table_idp_status_history_arn
     vpc_id                            = module.network.vpc_id
     vpc_subnet_ids                    = module.network.intra_subnets_ids
     vpc_s3_prefix_id                  = module.network.vpc_endpoints["s3"]["prefix_list_id"]
+    vpc_endpoint_dynamodb_prefix_id   = module.network.vpc_endpoints["dynamodb"]["prefix_list_id"]
     cloudwatch_logs_retention_in_days = var.lambda_cloudwatch_logs_retention_in_days
     environment_variables = {
       LOG_LEVEL                 = var.app_log_level
       IDP_STATUS_DYNAMODB_TABLE = module.database.table_idp_status_history_name
+      IDP_STATUS_DYNAMODB_IDX   = module.database.table_idp_status_history_idx_name
       ASSETS_S3_BUCKET          = module.storage.assets_bucket_name
       IDP_STATUS_S3_FILE_NAME   = "idp_status_history.json"
     }

src/infra/uat/eu-south-1/main.tf

@@ -293,6 +293,10 @@ module "backend" {
     gsi_pointer_arn = module.database.table_idpMetadata_gsi_pointer_arn
     table_arn       = module.database.table_idp_metadata_arn
   }
+  dynamodb_table_idpStatus = {
+    gsi_pointer_arn = module.database.table_idp_status_gsi_pointer_arn
+    table_arn       = module.database.table_idp_status_history_arn
+  }
 
   is_gh_integration_lambda = {
     name                              = format("%s-is-gh-integration-lambda", local.project)
@@ -306,14 +310,15 @@ module "backend" {
     name                              = format("%s-update-idp-status", local.project)
     filename                          = "${path.module}/../../hello-python/lambda.zip"
     assets_bucket_arn                 = module.storage.assets_bucket_arn
-    table_idp_status_history_arn      = module.database.table_idp_status_history_arn
     vpc_id                            = module.network.vpc_id
     vpc_subnet_ids                    = module.network.intra_subnets_ids
     vpc_s3_prefix_id                  = module.network.vpc_endpoints["s3"]["prefix_list_id"]
+    vpc_endpoint_dynamodb_prefix_id   = module.network.vpc_endpoints["dynamodb"]["prefix_list_id"]
     cloudwatch_logs_retention_in_days = var.lambda_cloudwatch_logs_retention_in_days
     environment_variables = {
       LOG_LEVEL                 = var.app_log_level
       IDP_STATUS_DYNAMODB_TABLE = module.database.table_idp_status_history_name
+      IDP_STATUS_DYNAMODB_IDX   = module.database.table_idp_status_history_idx_name
       ASSETS_S3_BUCKET          = module.storage.assets_bucket_name
       IDP_STATUS_S3_FILE_NAME   = "idp_status_history.json"
     }

src/infra/uat/eu-south-1/variables.tf

@@ -394,18 +394,12 @@ variable "client_ids" {
 variable "entity_id" {
   type = list(string)
   default = [
-    "https://loginspid.infocamere.it",
-    "https://idp.intesigroup.com",
-    "https://loginspid.aruba.it",
-    "https://identity.sieltecloud.it",
-    "https://spid.register.it",
-    "https://spid.teamsystem.com/idp",
-    "https://idp.namirialtsp.com/idp",
-    "https://posteid.poste.it",
-    "https://identity.infocert.it",
-    "https://id.eht.eu",
-    "https://login.id.tim.it/affwebservices/public/saml2sso",
-    "https://id.lepida.it/idp/shibboleth",
+    "https://demo.spid.gov.it",
+    "https://validator.spid.gov.it",
+    "https://validator.dev.oneid.pagopa.it",
+    "https://5ucp2co2zvqle6tcyrx4i5se7q0xdkni.lambda-url.eu-south-1.on.aws",
+    "https://validator.dev.oneid.pagopa.it/demo",
+    "https://koz3yhpkscymaqgp4m7ceguu6m0tffuz.lambda-url.eu-south-1.on.aws",
   ]
 }
 

src/oneid/oneid-lambda-update-idp-status/lambda.py

@@ -11,6 +11,7 @@
 
 AWS_REGION = os.getenv("AWS_REGION")
 IDP_STATUS_DYNAMODB_TABLE = os.getenv("IDP_STATUS_DYNAMODB_TABLE")
+IDP_STATUS_DYNAMODB_IDX = os.getenv("IDP_STATUS_DYNAMODB_IDX")
 ASSETS_S3_BUCKET = os.getenv("ASSETS_S3_BUCKET")
 IDP_STATUS_S3_FILE_NAME = os.getenv("IDP_STATUS_S3_FILE_NAME")
 
@@ -116,9 +117,10 @@ def get_all_latest_status():
     Get all items with {LATEST_POINTER} as the sort key from DynamoDB
     """
     try:
-        response = dynamodb_client.scan(
+        response = dynamodb_client.query(
             TableName=IDP_STATUS_DYNAMODB_TABLE,
-            FilterExpression="pointer = :pointer",
+            IndexName=IDP_STATUS_DYNAMODB_IDX,
+            KeyConditionExpression="pointer = :pointer",
             ExpressionAttributeValues={":pointer": {"S": LATEST_POINTER}},
         )
         return response.get("Items", [])