From 445d2d6972fdf8b6bc76cd43fb7e39351b2aa347 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Tue, 16 Jul 2024 13:26:20 -0700 Subject: [PATCH 1/9] fix: Use hyphens instead of underscores in TF module names --- .../{sns_sqs_lambda => sns-sqs-lambda}/.terraform.lock.hcl | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/README.md | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/data.tf | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/locals.tf | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/main.tf | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/output.tf | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/variables.tf | 0 .../evaluators/{sns_sqs_lambda => sns-sqs-lambda}/versions.tf | 0 .../.terraform.lock.hcl | 0 .../{s3_bucket_notification => s3-bucket-notification}/README.md | 0 .../{s3_bucket_notification => s3-bucket-notification}/main.tf | 0 .../variables.tf | 0 .../versions.tf | 0 .../{scheduled_task => scheduled-task}/.terraform.lock.hcl | 0 .../triggers/{scheduled_task => scheduled-task}/README.md | 0 .../triggers/{scheduled_task => scheduled-task}/data.tf | 0 .../triggers/{scheduled_task => scheduled-task}/locals.tf | 0 .../triggers/{scheduled_task => scheduled-task}/main.tf | 0 .../{scheduled_task => scheduled-task}/main.tf.cloudwatch_event | 0 .../triggers/{scheduled_task => scheduled-task}/variables.tf | 0 .../triggers/{scheduled_task => scheduled-task}/versions.tf | 0 21 files changed, 0 insertions(+), 0 deletions(-) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/.terraform.lock.hcl (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/README.md (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/data.tf (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/locals.tf (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/main.tf (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/output.tf (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/variables.tf (100%) rename terraform-unity/evaluators/{sns_sqs_lambda => sns-sqs-lambda}/versions.tf (100%) rename terraform-unity/triggers/{s3_bucket_notification => s3-bucket-notification}/.terraform.lock.hcl (100%) rename terraform-unity/triggers/{s3_bucket_notification => s3-bucket-notification}/README.md (100%) rename terraform-unity/triggers/{s3_bucket_notification => s3-bucket-notification}/main.tf (100%) rename terraform-unity/triggers/{s3_bucket_notification => s3-bucket-notification}/variables.tf (100%) rename terraform-unity/triggers/{s3_bucket_notification => s3-bucket-notification}/versions.tf (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/.terraform.lock.hcl (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/README.md (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/data.tf (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/locals.tf (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/main.tf (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/main.tf.cloudwatch_event (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/variables.tf (100%) rename terraform-unity/triggers/{scheduled_task => scheduled-task}/versions.tf (100%) diff --git a/terraform-unity/evaluators/sns_sqs_lambda/.terraform.lock.hcl b/terraform-unity/evaluators/sns-sqs-lambda/.terraform.lock.hcl similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/.terraform.lock.hcl rename to terraform-unity/evaluators/sns-sqs-lambda/.terraform.lock.hcl diff --git a/terraform-unity/evaluators/sns_sqs_lambda/README.md b/terraform-unity/evaluators/sns-sqs-lambda/README.md similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/README.md rename to terraform-unity/evaluators/sns-sqs-lambda/README.md diff --git a/terraform-unity/evaluators/sns_sqs_lambda/data.tf b/terraform-unity/evaluators/sns-sqs-lambda/data.tf similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/data.tf rename to terraform-unity/evaluators/sns-sqs-lambda/data.tf diff --git a/terraform-unity/evaluators/sns_sqs_lambda/locals.tf b/terraform-unity/evaluators/sns-sqs-lambda/locals.tf similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/locals.tf rename to terraform-unity/evaluators/sns-sqs-lambda/locals.tf diff --git a/terraform-unity/evaluators/sns_sqs_lambda/main.tf b/terraform-unity/evaluators/sns-sqs-lambda/main.tf similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/main.tf rename to terraform-unity/evaluators/sns-sqs-lambda/main.tf diff --git a/terraform-unity/evaluators/sns_sqs_lambda/output.tf b/terraform-unity/evaluators/sns-sqs-lambda/output.tf similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/output.tf rename to terraform-unity/evaluators/sns-sqs-lambda/output.tf diff --git a/terraform-unity/evaluators/sns_sqs_lambda/variables.tf b/terraform-unity/evaluators/sns-sqs-lambda/variables.tf similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/variables.tf rename to terraform-unity/evaluators/sns-sqs-lambda/variables.tf diff --git a/terraform-unity/evaluators/sns_sqs_lambda/versions.tf b/terraform-unity/evaluators/sns-sqs-lambda/versions.tf similarity index 100% rename from terraform-unity/evaluators/sns_sqs_lambda/versions.tf rename to terraform-unity/evaluators/sns-sqs-lambda/versions.tf diff --git a/terraform-unity/triggers/s3_bucket_notification/.terraform.lock.hcl b/terraform-unity/triggers/s3-bucket-notification/.terraform.lock.hcl similarity index 100% rename from terraform-unity/triggers/s3_bucket_notification/.terraform.lock.hcl rename to terraform-unity/triggers/s3-bucket-notification/.terraform.lock.hcl diff --git a/terraform-unity/triggers/s3_bucket_notification/README.md b/terraform-unity/triggers/s3-bucket-notification/README.md similarity index 100% rename from terraform-unity/triggers/s3_bucket_notification/README.md rename to terraform-unity/triggers/s3-bucket-notification/README.md diff --git a/terraform-unity/triggers/s3_bucket_notification/main.tf b/terraform-unity/triggers/s3-bucket-notification/main.tf similarity index 100% rename from terraform-unity/triggers/s3_bucket_notification/main.tf rename to terraform-unity/triggers/s3-bucket-notification/main.tf diff --git a/terraform-unity/triggers/s3_bucket_notification/variables.tf b/terraform-unity/triggers/s3-bucket-notification/variables.tf similarity index 100% rename from terraform-unity/triggers/s3_bucket_notification/variables.tf rename to terraform-unity/triggers/s3-bucket-notification/variables.tf diff --git a/terraform-unity/triggers/s3_bucket_notification/versions.tf b/terraform-unity/triggers/s3-bucket-notification/versions.tf similarity index 100% rename from terraform-unity/triggers/s3_bucket_notification/versions.tf rename to terraform-unity/triggers/s3-bucket-notification/versions.tf diff --git a/terraform-unity/triggers/scheduled_task/.terraform.lock.hcl b/terraform-unity/triggers/scheduled-task/.terraform.lock.hcl similarity index 100% rename from terraform-unity/triggers/scheduled_task/.terraform.lock.hcl rename to terraform-unity/triggers/scheduled-task/.terraform.lock.hcl diff --git a/terraform-unity/triggers/scheduled_task/README.md b/terraform-unity/triggers/scheduled-task/README.md similarity index 100% rename from terraform-unity/triggers/scheduled_task/README.md rename to terraform-unity/triggers/scheduled-task/README.md diff --git a/terraform-unity/triggers/scheduled_task/data.tf b/terraform-unity/triggers/scheduled-task/data.tf similarity index 100% rename from terraform-unity/triggers/scheduled_task/data.tf rename to terraform-unity/triggers/scheduled-task/data.tf diff --git a/terraform-unity/triggers/scheduled_task/locals.tf b/terraform-unity/triggers/scheduled-task/locals.tf similarity index 100% rename from terraform-unity/triggers/scheduled_task/locals.tf rename to terraform-unity/triggers/scheduled-task/locals.tf diff --git a/terraform-unity/triggers/scheduled_task/main.tf b/terraform-unity/triggers/scheduled-task/main.tf similarity index 100% rename from terraform-unity/triggers/scheduled_task/main.tf rename to terraform-unity/triggers/scheduled-task/main.tf diff --git a/terraform-unity/triggers/scheduled_task/main.tf.cloudwatch_event b/terraform-unity/triggers/scheduled-task/main.tf.cloudwatch_event similarity index 100% rename from terraform-unity/triggers/scheduled_task/main.tf.cloudwatch_event rename to terraform-unity/triggers/scheduled-task/main.tf.cloudwatch_event diff --git a/terraform-unity/triggers/scheduled_task/variables.tf b/terraform-unity/triggers/scheduled-task/variables.tf similarity index 100% rename from terraform-unity/triggers/scheduled_task/variables.tf rename to terraform-unity/triggers/scheduled-task/variables.tf diff --git a/terraform-unity/triggers/scheduled_task/versions.tf b/terraform-unity/triggers/scheduled-task/versions.tf similarity index 100% rename from terraform-unity/triggers/scheduled_task/versions.tf rename to terraform-unity/triggers/scheduled-task/versions.tf From fa1db111a1082a90ad8440c8cd385cdc2fb44a39 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Tue, 16 Jul 2024 13:42:46 -0700 Subject: [PATCH 2/9] chore: Bump TF version --- terraform-unity/evaluators/sns-sqs-lambda/README.md | 2 +- terraform-unity/evaluators/sns-sqs-lambda/versions.tf | 2 +- terraform-unity/initiator/README.md | 2 +- terraform-unity/initiator/versions.tf | 2 +- terraform-unity/triggers/s3-bucket-notification/README.md | 2 +- terraform-unity/triggers/scheduled-task/README.md | 2 +- terraform-unity/triggers/scheduled-task/versions.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/terraform-unity/evaluators/sns-sqs-lambda/README.md b/terraform-unity/evaluators/sns-sqs-lambda/README.md index 646d830..857a710 100644 --- a/terraform-unity/evaluators/sns-sqs-lambda/README.md +++ b/terraform-unity/evaluators/sns-sqs-lambda/README.md @@ -5,7 +5,7 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1.4.6 | +| [terraform](#requirement\_terraform) | ~> 1.8.2 | | [archive](#requirement\_archive) | >=2.4.2 | | [aws](#requirement\_aws) | >=5.50.0 | | [local](#requirement\_local) | >=2.5.1 | diff --git a/terraform-unity/evaluators/sns-sqs-lambda/versions.tf b/terraform-unity/evaluators/sns-sqs-lambda/versions.tf index 5e8229b..9f64e30 100644 --- a/terraform-unity/evaluators/sns-sqs-lambda/versions.tf +++ b/terraform-unity/evaluators/sns-sqs-lambda/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.4.6" + required_version = "~> 1.8.2" required_providers { archive = { diff --git a/terraform-unity/initiator/README.md b/terraform-unity/initiator/README.md index 93fb2d9..cda2793 100644 --- a/terraform-unity/initiator/README.md +++ b/terraform-unity/initiator/README.md @@ -5,7 +5,7 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1.4.6 | +| [terraform](#requirement\_terraform) | ~> 1.8.2 | | [aws](#requirement\_aws) | >=5.50.0 | | [local](#requirement\_local) | >=2.5.1 | | [null](#requirement\_null) | >=3.2.2 | diff --git a/terraform-unity/initiator/versions.tf b/terraform-unity/initiator/versions.tf index 4523ecc..3c46abe 100644 --- a/terraform-unity/initiator/versions.tf +++ b/terraform-unity/initiator/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.4.6" + required_version = "~> 1.8.2" required_providers { aws = { diff --git a/terraform-unity/triggers/s3-bucket-notification/README.md b/terraform-unity/triggers/s3-bucket-notification/README.md index 414184b..14c7a4e 100644 --- a/terraform-unity/triggers/s3-bucket-notification/README.md +++ b/terraform-unity/triggers/s3-bucket-notification/README.md @@ -5,7 +5,7 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1.4.6 | +| [terraform](#requirement\_terraform) | ~> 1.8.2 | | [aws](#requirement\_aws) | >=5.50.0 | | [local](#requirement\_local) | >=2.5.1 | | [null](#requirement\_null) | >=3.2.2 | diff --git a/terraform-unity/triggers/scheduled-task/README.md b/terraform-unity/triggers/scheduled-task/README.md index 0d3d32f..9b132ce 100644 --- a/terraform-unity/triggers/scheduled-task/README.md +++ b/terraform-unity/triggers/scheduled-task/README.md @@ -5,7 +5,7 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1.4.6 | +| [terraform](#requirement\_terraform) | ~> 1.8.2 | | [archive](#requirement\_archive) | >=2.4.2 | | [aws](#requirement\_aws) | >=5.50.0 | | [local](#requirement\_local) | >=2.5.1 | diff --git a/terraform-unity/triggers/scheduled-task/versions.tf b/terraform-unity/triggers/scheduled-task/versions.tf index 5e8229b..9f64e30 100644 --- a/terraform-unity/triggers/scheduled-task/versions.tf +++ b/terraform-unity/triggers/scheduled-task/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.4.6" + required_version = "~> 1.8.2" required_providers { archive = { From 3850c77c345283c3392fc1bca93945f5ddcbe25e Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Tue, 16 Jul 2024 14:32:29 -0700 Subject: [PATCH 3/9] feat: Change router input to be S3 URL --- terraform-unity/initiator/README.md | 4 +--- terraform-unity/initiator/main.tf | 14 +++----------- terraform-unity/initiator/variables.tf | 9 ++------- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/terraform-unity/initiator/README.md b/terraform-unity/initiator/README.md index cda2793..ba1abfe 100644 --- a/terraform-unity/initiator/README.md +++ b/terraform-unity/initiator/README.md @@ -34,7 +34,6 @@ No modules. | [aws_lambda_event_source_mapping.initiator_queue_event_source_mapping](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource | | [aws_lambda_function.initiator_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource | | [aws_s3_object.lambda_package](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource | -| [aws_s3_object.router_config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource | | [aws_sns_topic.initiator_topic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource | | [aws_sns_topic_subscription.initiator_subscription](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource | | [aws_sqs_queue.initiator_dead_letter_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | @@ -50,10 +49,9 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes | -| [config\_bucket](#input\_config\_bucket) | The S3 bucket where router configuration files will be stored and accessed | `string` | n/a | yes | | [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes | | [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no | -| [router\_config](#input\_router\_config) | The local path to the router configuration file to use | `string` | n/a | yes | +| [router\_config](#input\_router\_config) | The S3 URL to the router configuration file | `string` | n/a | yes | | [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no | ## Outputs diff --git a/terraform-unity/initiator/main.tf b/terraform-unity/initiator/main.tf index 9572f55..abbd37a 100644 --- a/terraform-unity/initiator/main.tf +++ b/terraform-unity/initiator/main.tf @@ -17,16 +17,8 @@ resource "aws_s3_object" "lambda_package" { tags = local.tags } -resource "aws_s3_object" "router_config" { - bucket = var.config_bucket - key = basename(var.router_config) - source = var.router_config - etag = filemd5(var.router_config) - tags = local.tags -} - resource "aws_lambda_function" "initiator_lambda" { - depends_on = [aws_s3_object.lambda_package, aws_s3_object.router_config] + depends_on = [aws_s3_object.lambda_package] function_name = "${var.project}-${var.venue}-${var.deployment_name}-inititator" s3_bucket = var.code_bucket s3_key = "unity_initiator-${jsondecode(data.local_file.version.content).version}-lambda.zip" @@ -37,7 +29,7 @@ resource "aws_lambda_function" "initiator_lambda" { environment { variables = { - ROUTER_CFG_URL = "s3://${aws_s3_object.router_config.bucket}/${aws_s3_object.router_config.key}" + ROUTER_CFG_URL = var.router_config } } tags = local.tags @@ -178,4 +170,4 @@ resource "aws_lambda_event_source_mapping" "initiator_queue_event_source_mapping enabled = true event_source_arn = aws_sqs_queue.initiator_queue.arn function_name = aws_lambda_function.initiator_lambda.arn -} \ No newline at end of file +} diff --git a/terraform-unity/initiator/variables.tf b/terraform-unity/initiator/variables.tf index 296e4f7..5e40f02 100644 --- a/terraform-unity/initiator/variables.tf +++ b/terraform-unity/initiator/variables.tf @@ -20,12 +20,7 @@ variable "code_bucket" { type = string } -variable "config_bucket" { - description = "The S3 bucket where router configuration files will be stored and accessed" - type = string -} - variable "router_config" { - description = "The local path to the router configuration file to use" + description = "The S3 URL to the router configuration file" type = string -} \ No newline at end of file +} From f22fbdcc6831f483ad2f9e0b21a79274a1a352f1 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Tue, 16 Jul 2024 14:50:46 -0700 Subject: [PATCH 4/9] chore: Use realpath in docker run call --- terraform-unity/initiator/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform-unity/initiator/main.tf b/terraform-unity/initiator/main.tf index abbd37a..7a4934e 100644 --- a/terraform-unity/initiator/main.tf +++ b/terraform-unity/initiator/main.tf @@ -3,7 +3,7 @@ resource "null_resource" "build_lambda_package" { provisioner "local-exec" { command = < Date: Tue, 16 Jul 2024 17:45:36 -0700 Subject: [PATCH 5/9] fix: Remove !!python/regexp to make yaml valid --- README.md | 108 ++++++++++++++++-- .../resources/routers_schema.yaml | 4 +- src/unity_initiator/router.py | 10 ++ tests/resources/test_bad_router_3.yaml | 2 +- tests/resources/test_bad_router_4.yaml | 2 +- tests/resources/test_bad_router_5.yaml | 2 +- tests/resources/test_bad_router_6.yaml | 2 +- tests/resources/test_bad_router_7.yaml | 2 +- tests/resources/test_router.yaml | 14 +-- 9 files changed, 121 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index eeb3406..057fd19 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,14 @@ ### What is Unity SDS? + Quite simply, an SDS (Science Data System) is an orchestrated set of networked compute and storage resources that is adapted to process science data through a pipeline. As described by [Hua et al. [2022]](#1): > Science Data Systems (SDSes) provide the capability to develop, test, process, and analyze instrument observational data efficiently, systematically, and at large scales. SDSes ingest the raw satellite instrument observations and process them from low‐level instrument values into higher level observational measurement values that compose the science data products. The [Unity SDS](https://github.com/unity-sds) is an implementation of an SDS by the Unity project at NASA Jet Propulsion Laboratory. ### What are triggers? + Trigger events are events that could potentially kick off processing in an SDS. Examples of trigger events are: 1. A raw data file is deposited into a location e.g. an S3 bucket or a local directory. @@ -43,10 +45,11 @@ These are just an initial subset of the different types of trigger events and th Trigger events by themselves don't automatically mean that SDS processing is ready to proceed. That's what evaluators are for. ### What are evaluators? + As described by [Hua et al. [2022]](#1): > A fundamental capability of an SDS is to systematically process science data through a series of data transformations from raw instrument data to geophysical measurements. Data are first made available to the SDS from GDS to be processed to higher level data products. The data transformation steps may utilize ancillary and auxiliary files as well as production rules that stipulate conditions for when each step should be executed. -In an SDS, evaluators are functions (irrespective of how they are deployed and called) that perform adaptation-specific evaluation to determine if the next step in the processing pipeline is ready for execution. +In an SDS, evaluators are functions (irrespective of how they are deployed and called) that perform adaptation-specific evaluation to determine if the next step in the processing pipeline is ready for execution. As an example, the following shows the input-output diagram for the NISAR L-SAR L0B PGE (a.k.a. science algorithm): @@ -72,6 +75,7 @@ The following screenshot shows examples of both of these interfaces: It is the responsibility of the initiator to perform the routing of triggers to their respective evaluators. ### What is the Unity initiator? + The Unity initiator is the set of compute resources that enable the routing of trigger events to their respective evaluators. It is agnostic of the trigger event source and agnostic of the adaptation-specific evaluator code. It is completely driven by configuration (a.k.a. router configuration YAML). The following screenshot shows the current architecture for the initiator: ![initiator](https://github.com/unity-sds/unity-initiator/assets/387300/74f7c2cb-8542-4ad8-9212-e720077373c0) @@ -79,6 +83,7 @@ The Unity initiator is the set of compute resources that enable the routing of t The initiator topic, an SNS topic, is the common interface that all triggers will submit events to. The initiator topic is subscribed to by the initiator SQS queue (complete with dead-letter queue for resiliency) which in turn is subscribed to by the router Lambda function. How the router Lambda routes payloads of the trigger events is defined by the router configuration YAML. The full YAML schema for the router configuration is located [here](src/unity_initiator/resources/routers_schema.yaml). #### How the router works + In the context of trigger events where a new file is detected (payload_type=`url`), the router Lambda extracts the URL of the new file, instantiates a router object and attempts to match it up against of set of regular expressions defined in the router configuration file. Let's consider this minimal router configuration YAML file example: ``` @@ -87,7 +92,7 @@ initiator_config: payload_type: url: - regexes: - - !!python/regexp '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{6})\d{3}\.vc(?P\w{2}))$' + - '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{6})\d{3}\.vc(?P\w{2}))$' evaluators: - name: eval_nisar_ingest actions: @@ -105,6 +110,7 @@ initiator_config: ``` and a trigger event payload for a new file that was triggered: + ``` { "payload": "s3://test_bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc29" @@ -113,14 +119,15 @@ and a trigger event payload for a new file that was triggered: The router will iterate over the set of url configs and attempt to match the URL against its set of regexes. If a match is successful, the router will iterate over the configured evaluators configs and perform the configured action to submit the URL payload to the evaluator interface (either SNS topic or DAG submission). In this case, the router sees that the action is `submit_to_sns_topic` and thus publishes the URL payload (and the regular expression captured groups as `payload_info`) to the SNS topic (`topic_arn`) configured in the action's parameters. In addition to the payload URL and the payload info, the router also includes the `on_success` parameters configured for the action. This will propagate pertinent info to the underlying evaluator code which would be used if evaluation is successful. In this case, if the evaulator successfully evaluates that everything is ready for this input file, it can proceed to submit a DAG run for the `submit_nisar_tlm_ingest` DAG in the underlying SPS. -Let's consider another example but this time the configured action is to submit a DAG run instead of publishing to an evaluator's SNS topic: +Let's consider another example but this time the configured action is to submit a DAG run instead of publishing to an evaluator's SNS topic: + ``` initiator_config: name: minimal config example payload_type: url: - regexes: - - !!python/regexp '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{5})(?P\d{1,4})\.ldf)$' + - '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{5})(?P\d{1,4})\.ldf)$' evaluators: - name: eval_nisar_l0a_readiness actions: @@ -143,6 +150,7 @@ initiator_config: ``` and a trigger event payload for a new file that was triggered: + ``` { "payload": "s3://test_bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.ldf" @@ -168,10 +176,13 @@ In this case, the router sees that the action is `submit_dag_by_id` and thus mak ## Contents +* [Features](#features) +* [Contents](#contents) * [Quick Start](#quick-start) + * [Requirements](#requirements) * [Setting Up the End-to-End Demo](#setting-up-the-end-to-end-demo) - * [Deploying the Inititator](#deploying-the-initiator) - * [Deploying an Example Evaluator](#deploying-an-example-evaluator-sns-topic-sqs-queue-lambda) + * [Deploying the Initiator](#deploying-the-initiator) + * [Deploying an Example Evaluator (SNS topic-\>SQS queue-\>Lambda)](#deploying-an-example-evaluator-sns-topic-sqs-queue-lambda) * [Deploying an S3 Event Notification Trigger](#deploying-an-s3-event-notification-trigger) * [Verify End-to-End Functionality (part 1)](#verify-end-to-end-functionality-part-1) * [Deploying an EventBridge Scheduler Trigger](#deploying-an-eventbridge-scheduler-trigger) @@ -181,8 +192,8 @@ In this case, the router sees that the action is `submit_dag_by_id` and thus mak * [Build Instructions](#build-instructions) * [Test Instructions](#test-instructions) * [Changelog](#changelog) -* [FAQ](#frequently-asked-questions-faq) -* [Contributing Guide](#contributing) +* [Frequently Asked Questions (FAQ)](#frequently-asked-questions-faq) +* [Contributing](#contributing) * [License](#license) * [References](#references) @@ -205,14 +216,19 @@ This guide provides a quick way to get started with our project. Please see our #### Deploying the Initiator 1. Clone repo: + ``` git clone https://github.com/unity-sds/unity-initiator.git ``` + 1. Change directory to the location of the inititator terraform: + ``` cd unity-initiator/terraform-unity/initiator/ ``` + 1. Copy a sample router configuration YAML file to use for deployment and update the AWS region and AWS account ID to match your AWS environment. We will be using the NISAR TLM test case for this demo so we also rename the SNS topic ARN for it accordingly: + ``` cp ../../tests/resources/test_router.yaml . export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --output text | awk '{print $1}') @@ -220,23 +236,33 @@ This guide provides a quick way to get started with our project. Please see our sed -i "s/hilo-hawaii-1/${AWS_REGION}/g" test_router.yaml sed -i "s/123456789012:eval_nisar_ingest/${AWS_ACCOUNT_ID}:uod-dev-eval_nisar_ingest-evaluator_topic/g" test_router.yaml ``` + 1. You will need an S3 bucket for terraform to stage the router Lambda zip file during deployment. Create one or reuse an existing one and set an environment variable for it: + ``` export CODE_BUCKET= ``` + 1. You will need an S3 bucket to store the router configuration YAML file. Create one or reuse an existing one (could be the same one in the previous step) and set an environment variable for it: + ``` export CONFIG_BUCKET= ``` + 1. Set a deployment name: + ``` export DEPLOYMENT_NAME=gmanipon-test ``` + 1. Initialize terraform: + ``` terraform init ``` + 1. Run terraform apply: + ``` terraform apply \ --var deployment_name=${DEPLOYMENT_NAME} \ @@ -245,55 +271,79 @@ This guide provides a quick way to get started with our project. Please see our --var router_config=test_router.yaml \ -auto-approve ``` + **Take note of the `initiator_topic_arn` that is output by terraform. It will be used when setting up any triggers.** #### Deploying an Example Evaluator (SNS topic->SQS queue->Lambda) + 1. Change directory to the location of the sns_sqs_lambda evaluator terraform: + ``` cd ../evaluators/sns_sqs_lambda/ ``` + 1. Set the name of the evaluator to our NISAR example: + ``` export EVALUATOR_NAME=eval_nisar_ingest ``` + 1. Note the implementation of the evaluator code. It currently doesn't do any real evaluation but simply returns that evaluation was successful: + ``` cat data.tf ``` + 1. Initialize terraform: + ``` terraform init ``` + 1. Run terraform apply: + ``` terraform apply \ --var evaluator_name=${EVALUATOR_NAME} \ -auto-approve ``` + **Take note of the `evaluator_topic_arn` that is output by terraform. It should match the topic ARN in the test_router.yaml file you used during the initiator deployment. If they match then the router Lambda is now able to submit payloads to this evaluator SNS topic.** - + #### Deploying an S3 Event Notification Trigger + 1. Change directory to the location of the s3_bucket_notification trigger terraform: + ``` cd ../../triggers/s3_bucket_notification/ ``` + 1. You will need an S3 bucket to configure event notification on. Create one or reuse an existing one (could be the same one in the previous steps) and set an environment variable for it: + ``` export ISL_BUCKET= ``` + 1. Specify an S3 prefix from which S3 event notifications will be emitted when objects are created: + ``` export ISL_BUCKET_PREFIX=incoming/ ``` + 1. Export the `initiator_topic_arn` that was output from the initiator terraform deployment: + ``` export INITIATOR_TOPIC_ARN= ``` + 1. Initialize terraform: + ``` terraform init ``` + 1. Run terraform apply: + ``` terraform apply \ --var isl_bucket=${ISL_BUCKET} \ @@ -301,11 +351,14 @@ This guide provides a quick way to get started with our project. Please see our --var initiator_topic_arn=${INITIATOR_TOPIC_ARN} \ -auto-approve ``` + 1. Verify that the S3 event notification was correctly hooked up to the initiator by looking at the initiator Lambda's CloudWatch logs for a entry similar to this: ![cloudwatch_logs_s3_testevent](https://github.com/unity-sds/unity-initiator/assets/387300/460a0d0b-ee01-480d-afab-ba70185341fc) #### Verify End-to-End Functionality (part 1) + 1. Create some fake NISAR TLM files and stage them up to the ISL bucket under the ISL prefix: + ``` for i in $(echo 24 25 29); do echo 'Hawaii, No Ka Oi!' > NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc${i} @@ -313,23 +366,32 @@ This guide provides a quick way to get started with our project. Please see our rm NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc${i} done ``` + 1. Verify that the `eval_nisar_ingest` evaluator Lambda function was called successfully for each of those staged files by looking at its CloudWatch logs for entries similar to this: ![eval_log_1](https://github.com/unity-sds/unity-initiator/assets/387300/34a273a5-5992-46f8-982b-0a0ec37d1798) #### Deploying an EventBridge Scheduler Trigger + 1. Change directory to the location of the s3_bucket_notification trigger terraform: + ``` cd ../scheduled_task/ ``` + 1. Note the implementation of the trigger lambda code. It currently hard codes a payload URL however in a real implementation, code would be written to query for new files from some REST API, database, etc. Here we simulate that and simply return a NISAR TLM file: + ``` cat data.tf ``` + 1. Initialize terraform: + ``` terraform init ``` + 1. Run terraform apply. Note the DEPLOYMENT_NAME and INITIATOR_TOPIC_ARN environment variables should have been set in the previous steps. If not set them again: + ``` terraform apply \ --var deployment_name=${DEPLOYMENT_NAME} \ @@ -338,36 +400,49 @@ This guide provides a quick way to get started with our project. Please see our ``` #### Verify End-to-End Functionality (part 2) + 1. The deployed EventBridge scheduler runs the trigger Lambda function with schedule expression of `rate(1 minute)`. After a minute, verify that the `eval_nisar_ingest` evaluator Lambda function was called successfully for each of those scheduled invocations by looking at its CloudWatch logs for entries similar to this: ![eval_log_2](https://github.com/unity-sds/unity-initiator/assets/387300/cae82e10-a736-43b7-8957-790fc29b5fea) #### Tear Down + 1. Simply go back into each of the terraform directories for which `terraform apply` was run and run `terraform destroy`. - + ### Setup Instructions for Development 1. Clone repo: + ``` git clone https://github.com/unity-sds/unity-initiator.git ``` + 1. Install hatch: + ``` pip install hatch ``` + 1. Build virtualenv and install dependencies: + ``` cd unity-initiator hatch env create ``` + 1. Install dev tools: + ``` ./scripts/install_dev_tools.sh ``` + 1. Test pre-commit run: + ``` pre-commit run --all-files ``` + You should see the following output: + ``` check for merge conflicts...............................................................Passed check for broken symlinks...........................................(no files to check)Skipped @@ -391,14 +466,19 @@ This guide provides a quick way to get started with our project. Please see our 1. Follow [Setup Instructions for Development](#setup-instructions-for-development) above. 1. Enter environment: + ``` hatch shell ``` + 1. Build: + ``` hatch build ``` + Wheel and tarballs will be built in the `dist/` directory: + ``` $ tree dist dist @@ -414,14 +494,19 @@ This guide provides a quick way to get started with our project. Please see our 1. Follow [Setup Instructions for Development](#setup-instructions-for-development) above. 1. Enter environment: + ``` hatch shell ``` + 1. Run tests: + ``` hatch run pytest ``` + For more information during test runs, set the log level accordingly. For example: + ``` hatch run pytest -s -v --log-cli-level=INFO --log-level=INFO ``` @@ -455,8 +540,9 @@ For guidance on our governance approach, including decision-making process and o `unity-initiator` is distributed under the terms of the [MIT](https://spdx.org/licenses/MIT.html) license. ## References + [1] Hua, H., Manipon, G. and Shah, S. (2022). Scaling Big Earth Science Data Systems Via Cloud Computing. In Big Data Analytics in Earth, Atmospheric, and Ocean Sciences (eds T. Huang, T.C. Vance and C. Lynnes). -https://doi.org/10.1002/9781119467557.ch3 + diff --git a/src/unity_initiator/resources/routers_schema.yaml b/src/unity_initiator/resources/routers_schema.yaml index a53505d..c77d84b 100644 --- a/src/unity_initiator/resources/routers_schema.yaml +++ b/src/unity_initiator/resources/routers_schema.yaml @@ -25,7 +25,7 @@ initiator_config: # Configuration for matching payload (e.g. url) against a set of compiled regular expressions # and mapping any matches to a set of evaluators. regex_config: - regexes: list(compiled_regex(), required=True, min=1) + regexes: list(str, required=True, min=1) evaluators: list(include("evaluator_config"), required=True, min=1) # Configuration of actions that submit to evaluators. @@ -60,4 +60,4 @@ submit_to_sns_topic_action: # Configuration to pass onto the evaluator to use when evaluation is a success. on_success_actions: - actions: list(include("action_config"), required=True, min=1, max=1) \ No newline at end of file + actions: list(include("action_config"), required=True, min=1, max=1) diff --git a/src/unity_initiator/router.py b/src/unity_initiator/router.py index 60521b2..998fbb2 100644 --- a/src/unity_initiator/router.py +++ b/src/unity_initiator/router.py @@ -1,5 +1,6 @@ import asyncio import json +import re from .evaluator import Evaluator from .utils.conf_utils import YamlConf, YamlConfEncoder @@ -14,6 +15,15 @@ class Router: def __init__(self, config_file): self._config_file = config_file self._config = YamlConf(self._config_file) + self._compile_regexes() + + def _compile_regexes(self): + """Compile all regex strings in the configuration.""" + for url_cfg in ( + self._config.get("initiator_config").get("payload_type").get("url", []) + ): + if "regexes" in url_cfg: + url_cfg["regexes"] = [re.compile(regex) for regex in url_cfg["regexes"]] def get_evaluators_by_url(self, url): found_match = False diff --git a/tests/resources/test_bad_router_3.yaml b/tests/resources/test_bad_router_3.yaml index d706d8c..6db3719 100644 --- a/tests/resources/test_bad_router_3.yaml +++ b/tests/resources/test_bad_router_3.yaml @@ -3,5 +3,5 @@ initiator_config: payload_type: url: - regexes: - - !!python/regexp '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' + - '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' evaluators: diff --git a/tests/resources/test_bad_router_4.yaml b/tests/resources/test_bad_router_4.yaml index 0b3cd5f..9fb8927 100644 --- a/tests/resources/test_bad_router_4.yaml +++ b/tests/resources/test_bad_router_4.yaml @@ -3,7 +3,7 @@ initiator_config: payload_type: url: - regexes: - - !!python/regexp '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' + - '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' evaluators: - name: eval_sbg_l2_readiness actions: diff --git a/tests/resources/test_bad_router_5.yaml b/tests/resources/test_bad_router_5.yaml index e7fbf53..818de30 100644 --- a/tests/resources/test_bad_router_5.yaml +++ b/tests/resources/test_bad_router_5.yaml @@ -3,7 +3,7 @@ initiator_config: payload_type: url: - regexes: - - !!python/regexp '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' + - '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' evaluators: - name: eval_sbg_l2_readiness actions: diff --git a/tests/resources/test_bad_router_6.yaml b/tests/resources/test_bad_router_6.yaml index 7181560..7024c5b 100644 --- a/tests/resources/test_bad_router_6.yaml +++ b/tests/resources/test_bad_router_6.yaml @@ -3,7 +3,7 @@ initiator_config: payload_type: url: - regexes: - - !!python/regexp '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' + - '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P)_OBS\.bin)$' evaluators: - name: eval_sbg_l2_readiness actions: diff --git a/tests/resources/test_bad_router_7.yaml b/tests/resources/test_bad_router_7.yaml index 017a895..292f940 100644 --- a/tests/resources/test_bad_router_7.yaml +++ b/tests/resources/test_bad_router_7.yaml @@ -3,7 +3,7 @@ initiator_config: payload_type: url: - regexes: - - !!python/regexp '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{5})(?P\d{1,4})\.ldf)$' + - '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{5})(?P\d{1,4})\.ldf)$' evaluators: - name: eval_nisar_l0a_readiness actions: diff --git a/tests/resources/test_router.yaml b/tests/resources/test_router.yaml index 2f640ed..fcafc13 100644 --- a/tests/resources/test_router.yaml +++ b/tests/resources/test_router.yaml @@ -13,7 +13,7 @@ initiator_config: # SBG example: L1B data staged to S3 bucket and payload is S3 url - regexes: - - !!python/regexp '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P\d{3})_OBS\.bin)$' + - '/(?PSISTER_EMIT_L1B_RDN_(?P\d{8}T\d{6})_(?P\d{3})_OBS\.bin)$' evaluators: # If the regex matches, the router submits a JSON payload to the eval_sbg_l2_readiness SNS topic that contains @@ -52,9 +52,9 @@ initiator_config: # M2020 example: xyz left finder; example of matching any one of a set of regexes - regexes: - - !!python/regexp 'ids-pipeline/pipes/nonlin_xyz_left/inputque/.L.{17}_.{3}RAS_N.{26}\.VIC-link' - - !!python/regexp 'ids-pipeline/pipes/nonlin_xyz_left/inputque/.R.{17}_.{3}RAS_N.{26}\.VIC-link' - - !!python/regexp 'ids-pipeline/pipes/nonlin_xyz_left/inputque/.L.{17}_.{3}DSP_N.{26}\.VIC-link' + - 'ids-pipeline/pipes/nonlin_xyz_left/inputque/.L.{17}_.{3}RAS_N.{26}\.VIC-link' + - 'ids-pipeline/pipes/nonlin_xyz_left/inputque/.R.{17}_.{3}RAS_N.{26}\.VIC-link' + - 'ids-pipeline/pipes/nonlin_xyz_left/inputque/.L.{17}_.{3}DSP_N.{26}\.VIC-link' evaluators: # If any of the regexes match, the router submits a JSON payload to the eval_m2020_xyz_left_finder SNS topic that contains @@ -79,7 +79,7 @@ initiator_config: # NISAR example: GDS stages satellite telemetry to S3 bucket and payload is S3 url - regexes: - - !!python/regexp '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{6})\d{3}\.vc(?P\w{2}))$' + - '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{6})\d{3}\.vc(?P\w{2}))$' evaluators: # If the regex matches, the router submits a JSON payload to the eval_nisar_ingest SNS topic that contains @@ -104,7 +104,7 @@ initiator_config: # NISAR example: GDS stages LDF (list of delivered files) to S3 bucket and payload is S3 url - regexes: - - !!python/regexp '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{5})(?P\d{1,4})\.ldf)$' + - '/(?P(?PNISAR)_S(?P\d{3})_(?P\w{2,3})_(?P\w{3,4})_M(?P\d{2})_P(?P\d{5})_R(?P\d{2})_C(?P\d{2})_G(?P\d{2})_(?P\d{4}_\d{3}_\d{2}_\d{2}_\d{2}_\d{5})(?P\d{1,4})\.ldf)$' evaluators: # If the regex matches, the router submits a JSON payload to the eval_nisar_l0a_readiness DAG via Airflow REST API @@ -128,4 +128,4 @@ initiator_config: # Otherwise these can be uncommented out for explicit configuration (e.g. another SPS cluster) #airflow_base_api_endpoint: xxx #airflow_username: - #airflow_password: \ No newline at end of file + #airflow_password: From c263d60e66d8598001bfeeba3dcac6b4ae716d38 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Tue, 16 Jul 2024 17:52:58 -0700 Subject: [PATCH 6/9] Feat: Explicitly provision the lambda's cloudwatch log group --- terraform-unity/initiator/README.md | 1 + terraform-unity/initiator/main.tf | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/terraform-unity/initiator/README.md b/terraform-unity/initiator/README.md index ba1abfe..f3272e7 100644 --- a/terraform-unity/initiator/README.md +++ b/terraform-unity/initiator/README.md @@ -26,6 +26,7 @@ No modules. | Name | Type | |------|------| +| [aws_cloudwatch_log_group.initiator_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_iam_policy.initiator_lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.initiator_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | diff --git a/terraform-unity/initiator/main.tf b/terraform-unity/initiator/main.tf index 7a4934e..1452438 100644 --- a/terraform-unity/initiator/main.tf +++ b/terraform-unity/initiator/main.tf @@ -18,7 +18,7 @@ resource "aws_s3_object" "lambda_package" { } resource "aws_lambda_function" "initiator_lambda" { - depends_on = [aws_s3_object.lambda_package] + depends_on = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.initiator_lambda] function_name = "${var.project}-${var.venue}-${var.deployment_name}-inititator" s3_bucket = var.code_bucket s3_key = "unity_initiator-${jsondecode(data.local_file.version.content).version}-lambda.zip" @@ -35,6 +35,12 @@ resource "aws_lambda_function" "initiator_lambda" { tags = local.tags } +resource "aws_cloudwatch_log_group" "initiator_lambda" { + name = "/aws/lambda/${var.project}-${var.venue}-${var.deployment_name}-inititator" + retention_in_days = 14 + tags = local.tags +} + resource "aws_iam_role" "initiator_lambda_iam_role" { name = "${var.project}-${var.venue}-${var.deployment_name}-initiator_lambda_iam_role" From d5d0275c0e473db1f3625ee095304f04635bf136 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Wed, 17 Jul 2024 12:01:29 -0700 Subject: [PATCH 7/9] feat: New initiator action - OGC process execution --- .../actions/submit_ogc_process_execution.py | 50 +++++++++++++++++++ .../resources/routers_schema.yaml | 13 ++++- tests/resources/test_router.yaml | 15 ++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/unity_initiator/actions/submit_ogc_process_execution.py diff --git a/src/unity_initiator/actions/submit_ogc_process_execution.py b/src/unity_initiator/actions/submit_ogc_process_execution.py new file mode 100644 index 0000000..56df2ce --- /dev/null +++ b/src/unity_initiator/actions/submit_ogc_process_execution.py @@ -0,0 +1,50 @@ +import httpx + +from ..utils.logger import logger +from .base import Action + +__all__ = ["SubmitOgcProcessExecution"] + + +class SubmitOgcProcessExecution(Action): + def __init__(self, payload, payload_info, params): + super().__init__(payload, payload_info, params) + logger.info("instantiated %s", __class__.__name__) + + def execute(self): + logger.debug("executing execute in %s", __class__.__name__) + url = f"{self._params['ogc_processes_base_api_endpoint']}/processes/{self._params['process_id']}/execution" + logger.info("url: %s", url) + headers = {"Content-Type": "application/json", "Accept": "application/json"} + # body = { + # "inputs": self._params["execution_inputs"], + # "outputs": self._params["execution_outputs"], + # "subscriber": self._params["execution_subscriber"], + # } + body = { + "inputs": { + "payload": self._payload, + "payload_info": self._payload_info, + "on_success": self._params["on_success"], + }, + "outputs": None, + "subscriber": None, + } + response = httpx.post(url, headers=headers, json=body) + if response.status_code in (200, 201): + success = True + resp = response.json() + logger.info( + "Successfully triggered the execution of the OGC Process %s: %s", + self._params["process_id"], + resp, + ) + else: + success = False + resp = response.text + logger.info( + "Failed to trigger the execution of the OGC Process %s: %s", + self._params["process_id"], + resp, + ) + return {"success": success, "response": resp} diff --git a/src/unity_initiator/resources/routers_schema.yaml b/src/unity_initiator/resources/routers_schema.yaml index c77d84b..fdbabe9 100644 --- a/src/unity_initiator/resources/routers_schema.yaml +++ b/src/unity_initiator/resources/routers_schema.yaml @@ -36,7 +36,7 @@ evaluator_config: # Currently only 2 types of actions are supported: # 1. submit payload to an SNS topic # 2. submit payload to an airflow DAG -action_config: any(include("submit_dag_by_id_action"), include("submit_to_sns_topic_action")) +action_config: any(include("submit_dag_by_id_action"), include("submit_to_sns_topic_action"), include("submit_ogc_process_execution_action")) # Configuration for submitting a payload to an airflow DAG. submit_dag_by_id_action: @@ -58,6 +58,17 @@ submit_to_sns_topic_action: topic_arn: str(required=False) on_success: include("on_success_actions", required=False) +# Configuration for submitting a OGC process execution. +submit_ogc_process_execution_action: + name: str(equals="submit_ogc_process_execution") + params: + process_id: str() + ogc_processes_base_api_endpoint: str(required=False) + execution_inputs: map(required=False) + execution_outputs: map(required=False) + execution_subscriber: map(required=False) + on_success: include("on_success_actions", required=False) + # Configuration to pass onto the evaluator to use when evaluation is a success. on_success_actions: actions: list(include("action_config"), required=True, min=1, max=1) diff --git a/tests/resources/test_router.yaml b/tests/resources/test_router.yaml index fcafc13..c635646 100644 --- a/tests/resources/test_router.yaml +++ b/tests/resources/test_router.yaml @@ -129,3 +129,18 @@ initiator_config: #airflow_base_api_endpoint: xxx #airflow_username: #airflow_password: + + - regexes: + - '(?<=/)(?Phello_world\.txt)$' + evaluators: + - name: eval_hello_world_readiness + actions: + - name: submit_ogc_process_execution + params: + process_id: eval_hello_world_readiness + ogc_processes_base_api_endpoint: ${ogc_processes_base_api_endpoint} + on_success: + actions: + - name: submit_ogc_process_execution + params: + process_id: hello_world From 9040d922153f1902f60a23d05324d9061ee57cf9 Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Thu, 8 Aug 2024 14:11:05 -0700 Subject: [PATCH 8/9] chore: Update resource naming convention --- README.md | 58 +++++++++---------- terraform-unity/initiator/README.md | 1 - terraform-unity/initiator/locals.tf | 2 +- terraform-unity/initiator/main.tf | 8 +-- terraform-unity/initiator/variables.tf | 5 -- terraform-unity/triggers/cmr_query/README.md | 3 +- terraform-unity/triggers/cmr_query/locals.tf | 2 +- .../triggers/cmr_query/variables.tf | 5 -- .../triggers/cmr_query/versions.tf | 2 +- .../triggers/scheduled-task/README.md | 1 - .../triggers/scheduled-task/locals.tf | 2 +- .../triggers/scheduled-task/main.tf | 6 +- .../triggers/scheduled-task/variables.tf | 5 -- 13 files changed, 41 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 8d77fc6..b1e4b2d 100644 --- a/README.md +++ b/README.md @@ -177,28 +177,28 @@ In this case, the router sees that the action is `submit_dag_by_id` and thus mak ## Contents -- [Features](#features) -- [Contents](#contents) -- [Quick Start](#quick-start) - - [Requirements](#requirements) - - [Setting Up the End-to-End Demo](#setting-up-the-end-to-end-demo) - - [Deploying the Initiator](#deploying-the-initiator) - - [Deploying an Example Evaluator (SNS topic-\>SQS queue-\>Lambda)](#deploying-an-example-evaluator-sns-topic-sqs-queue-lambda) - - [Deploying an S3 Event Notification Trigger](#deploying-an-s3-event-notification-trigger) - - [Verify End-to-End Functionality (part 1)](#verify-end-to-end-functionality-part-1) - - [Deploying an EventBridge Scheduler Trigger](#deploying-an-eventbridge-scheduler-trigger) - - [Verify End-to-End Functionality (part 2)](#verify-end-to-end-functionality-part-2) - - [Deploying an EventBridge Scheduler Trigger for Periodic CMR Queries](#deploying-an-eventbridge-scheduler-trigger-for-periodic-cmr-queries) - - [Verify End-to-End Functionality (part 3)](#verify-end-to-end-functionality-part-3) - - [Tear Down](#tear-down) - - [Setup Instructions for Development](#setup-instructions-for-development) - - [Build Instructions](#build-instructions) - - [Test Instructions](#test-instructions) -- [Changelog](#changelog) -- [Frequently Asked Questions (FAQ)](#frequently-asked-questions-faq) -- [Contributing](#contributing) -- [License](#license) -- [References](#references) +* [Features](#features) +* [Contents](#contents) +* [Quick Start](#quick-start) + * [Requirements](#requirements) + * [Setting Up the End-to-End Demo](#setting-up-the-end-to-end-demo) + * [Deploying the Initiator](#deploying-the-initiator) + * [Deploying an Example Evaluator (SNS topic-\>SQS queue-\>Lambda)](#deploying-an-example-evaluator-sns-topic-sqs-queue-lambda) + * [Deploying an S3 Event Notification Trigger](#deploying-an-s3-event-notification-trigger) + * [Verify End-to-End Functionality (part 1)](#verify-end-to-end-functionality-part-1) + * [Deploying an EventBridge Scheduler Trigger](#deploying-an-eventbridge-scheduler-trigger) + * [Verify End-to-End Functionality (part 2)](#verify-end-to-end-functionality-part-2) + * [Deploying an EventBridge Scheduler Trigger for Periodic CMR Queries](#deploying-an-eventbridge-scheduler-trigger-for-periodic-cmr-queries) + * [Verify End-to-End Functionality (part 3)](#verify-end-to-end-functionality-part-3) + * [Tear Down](#tear-down) + * [Setup Instructions for Development](#setup-instructions-for-development) + * [Build Instructions](#build-instructions) + * [Test Instructions](#test-instructions) +* [Changelog](#changelog) +* [Frequently Asked Questions (FAQ)](#frequently-asked-questions-faq) +* [Contributing](#contributing) +* [License](#license) +* [References](#references) ## Quick Start @@ -253,10 +253,10 @@ This guide provides a quick way to get started with our project. Please see our export CONFIG_BUCKET= ``` -1. Set a deployment name: +1. Set a project name: ``` - export DEPLOYMENT_NAME=gmanipon-test + export PROJECT=gmanipon-test ``` 1. Initialize terraform: @@ -269,7 +269,7 @@ This guide provides a quick way to get started with our project. Please see our ``` terraform apply \ - --var deployment_name=${DEPLOYMENT_NAME} \ + --var project=${PROJECT} \ --var code_bucket=${CODE_BUCKET} \ --var config_bucket=${CONFIG_BUCKET} \ --var router_config=test_router.yaml \ @@ -400,11 +400,11 @@ This guide provides a quick way to get started with our project. Please see our terraform init ``` -1. Run terraform apply. Note the DEPLOYMENT_NAME and INITIATOR_TOPIC_ARN environment variables should have been set in the previous steps. If not set them again: +1. Run terraform apply. Note the PROJECT and INITIATOR_TOPIC_ARN environment variables should have been set in the previous steps. If not set them again: ``` terraform apply \ - --var deployment_name=${DEPLOYMENT_NAME} \ + --var project=${PROJECT} \ --var initiator_topic_arn=${INITIATOR_TOPIC_ARN} \ -auto-approve ``` @@ -452,11 +452,11 @@ This guide provides a quick way to get started with our project. Please see our terraform init ``` -1. Run terraform apply. Note the DEPLOYMENT_NAME, CODE_BUCKET and INITIATOR_TOPIC_ARN environment variables should have been set in the previous steps. If not set them again: +1. Run terraform apply. Note the PROJECT, CODE_BUCKET and INITIATOR_TOPIC_ARN environment variables should have been set in the previous steps. If not set them again: ``` terraform apply \ - --var deployment_name=${DEPLOYMENT_NAME} \ + --var project=${PROJECT} \ --var code_bucket=${CODE_BUCKET} \ --var initiator_topic_arn=${INITIATOR_TOPIC_ARN} \ --var provider_id=${PROVIDER_ID} \ diff --git a/terraform-unity/initiator/README.md b/terraform-unity/initiator/README.md index f3272e7..c2d0bc4 100644 --- a/terraform-unity/initiator/README.md +++ b/terraform-unity/initiator/README.md @@ -50,7 +50,6 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes | | [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no | | [router\_config](#input\_router\_config) | The S3 URL to the router configuration file | `string` | n/a | yes | | [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no | diff --git a/terraform-unity/initiator/locals.tf b/terraform-unity/initiator/locals.tf index a8f66e2..1ee28d9 100644 --- a/terraform-unity/initiator/locals.tf +++ b/terraform-unity/initiator/locals.tf @@ -1,5 +1,5 @@ locals { - function_name = "${var.project}-${var.venue}-${var.deployment_name}-inititator" + function_name = "${var.project}-${var.venue}-inititator" tags = { Venue = "dev" ServiceArea = "cs" diff --git a/terraform-unity/initiator/main.tf b/terraform-unity/initiator/main.tf index 967674e..36eacd6 100644 --- a/terraform-unity/initiator/main.tf +++ b/terraform-unity/initiator/main.tf @@ -19,7 +19,7 @@ resource "aws_s3_object" "lambda_package" { resource "aws_lambda_function" "initiator_lambda" { depends_on = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.initiator_lambda] - function_name = "${var.project}-${var.venue}-${var.deployment_name}-inititator" + function_name = "${var.project}-${var.venue}-inititator" s3_bucket = var.code_bucket s3_key = "unity_initiator-${jsondecode(data.local_file.version.content).version}-lambda.zip" handler = "unity_initiator.cloud.lambda_handler.lambda_handler_initiator" @@ -36,13 +36,13 @@ resource "aws_lambda_function" "initiator_lambda" { } resource "aws_cloudwatch_log_group" "initiator_lambda" { - name = "/aws/lambda/${var.project}-${var.venue}-${var.deployment_name}-inititator" + name = "/aws/lambda/${var.project}-${var.venue}-inititator" retention_in_days = 14 tags = local.tags } resource "aws_iam_role" "initiator_lambda_iam_role" { - name = "${var.project}-${var.venue}-${var.deployment_name}-initiator_lambda_iam_role" + name = "${var.project}-${var.venue}-initiator_lambda_iam_role" assume_role_policy = jsonencode({ Version = "2012-10-17", @@ -61,7 +61,7 @@ resource "aws_iam_role" "initiator_lambda_iam_role" { } resource "aws_iam_policy" "initiator_lambda_policy" { - name = "${var.project}-${var.venue}-${var.deployment_name}-initiator_lambda_policy" + name = "${var.project}-${var.venue}-initiator_lambda_policy" description = "A policy for the initiator lambda function to access S3 and SQS" policy = jsonencode({ diff --git a/terraform-unity/initiator/variables.tf b/terraform-unity/initiator/variables.tf index 5e40f02..e405c69 100644 --- a/terraform-unity/initiator/variables.tf +++ b/terraform-unity/initiator/variables.tf @@ -1,8 +1,3 @@ -variable "deployment_name" { - description = "The deployment name" - type = string -} - variable "project" { description = "The unity project its installed into" type = string diff --git a/terraform-unity/triggers/cmr_query/README.md b/terraform-unity/triggers/cmr_query/README.md index d894b0c..61745ca 100644 --- a/terraform-unity/triggers/cmr_query/README.md +++ b/terraform-unity/triggers/cmr_query/README.md @@ -5,7 +5,7 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1.4.6 | +| [terraform](#requirement\_terraform) | ~> 1.8.2 | | [archive](#requirement\_archive) | >=2.4.2 | | [aws](#requirement\_aws) | >=5.50.0 | | [local](#requirement\_local) | >=2.5.1 | @@ -52,7 +52,6 @@ No modules. |------|-------------|------|---------|:--------:| | [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes | | [concept\_id](#input\_concept\_id) | The concept ID for the data collection: https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#granule-search-by-parameters | `string` | n/a | yes | -| [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes | | [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes | | [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no | | [provider\_id](#input\_provider\_id) | The short name for the data provider: https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#granule-search-by-parameters | `string` | n/a | yes | diff --git a/terraform-unity/triggers/cmr_query/locals.tf b/terraform-unity/triggers/cmr_query/locals.tf index a7a381e..546c83d 100644 --- a/terraform-unity/triggers/cmr_query/locals.tf +++ b/terraform-unity/triggers/cmr_query/locals.tf @@ -1,5 +1,5 @@ locals { - function_name = "${var.project}-${var.venue}-${var.deployment_name}-cmr_query" + function_name = "${var.project}-${var.venue}-cmr_query" tags = { Venue = "dev" ServiceArea = "cs" diff --git a/terraform-unity/triggers/cmr_query/variables.tf b/terraform-unity/triggers/cmr_query/variables.tf index 2d138e9..1736fb1 100644 --- a/terraform-unity/triggers/cmr_query/variables.tf +++ b/terraform-unity/triggers/cmr_query/variables.tf @@ -1,8 +1,3 @@ -variable "deployment_name" { - description = "The deployment name" - type = string -} - variable "project" { description = "The unity project its installed into" type = string diff --git a/terraform-unity/triggers/cmr_query/versions.tf b/terraform-unity/triggers/cmr_query/versions.tf index 5e8229b..9f64e30 100644 --- a/terraform-unity/triggers/cmr_query/versions.tf +++ b/terraform-unity/triggers/cmr_query/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.4.6" + required_version = "~> 1.8.2" required_providers { archive = { diff --git a/terraform-unity/triggers/scheduled-task/README.md b/terraform-unity/triggers/scheduled-task/README.md index 02aae43..97d3743 100644 --- a/terraform-unity/triggers/scheduled-task/README.md +++ b/terraform-unity/triggers/scheduled-task/README.md @@ -42,7 +42,6 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes | | [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes | | [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no | | [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no | diff --git a/terraform-unity/triggers/scheduled-task/locals.tf b/terraform-unity/triggers/scheduled-task/locals.tf index 68396e8..d7997ca 100644 --- a/terraform-unity/triggers/scheduled-task/locals.tf +++ b/terraform-unity/triggers/scheduled-task/locals.tf @@ -1,5 +1,5 @@ locals { - function_name = "${var.project}-${var.venue}-${var.deployment_name}-scheduled_task" + function_name = "${var.project}-${var.venue}-scheduled_task" tags = { Venue = "dev" ServiceArea = "cs" diff --git a/terraform-unity/triggers/scheduled-task/main.tf b/terraform-unity/triggers/scheduled-task/main.tf index 78e57cf..16eea3b 100644 --- a/terraform-unity/triggers/scheduled-task/main.tf +++ b/terraform-unity/triggers/scheduled-task/main.tf @@ -50,7 +50,7 @@ resource "aws_cloudwatch_log_group" "scheduled_task_lambda_log_group" { } resource "aws_iam_role" "scheduler" { - name = "${var.project}-${var.venue}-${var.deployment_name}-cron-scheduler-role" + name = "${var.project}-${var.venue}-cron-scheduler-role" assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [ @@ -68,7 +68,7 @@ resource "aws_iam_role" "scheduler" { } resource "aws_iam_policy" "scheduler" { - name = "${var.project}-${var.venue}-${var.deployment_name}-cron-scheduler-policy" + name = "${var.project}-${var.venue}-cron-scheduler-policy" policy = jsonencode({ Version = "2012-10-17" Statement = [ @@ -90,7 +90,7 @@ resource "aws_iam_role_policy_attachment" "scheduler" { } resource "aws_scheduler_schedule" "run_scheduled_task" { - name = "${var.project}-${var.venue}-${var.deployment_name}-run_scheduled_task" + name = "${var.project}-${var.venue}-run_scheduled_task" schedule_expression = "rate(1 minute)" flexible_time_window { mode = "OFF" diff --git a/terraform-unity/triggers/scheduled-task/variables.tf b/terraform-unity/triggers/scheduled-task/variables.tf index f71ffef..495a65c 100644 --- a/terraform-unity/triggers/scheduled-task/variables.tf +++ b/terraform-unity/triggers/scheduled-task/variables.tf @@ -1,8 +1,3 @@ -variable "deployment_name" { - description = "The deployment name" - type = string -} - variable "project" { description = "The unity project its installed into" type = string From 4a9af0368677813c9302c728f169110daa24818d Mon Sep 17 00:00:00 2001 From: Drew Meyers Date: Mon, 12 Aug 2024 13:08:18 -0700 Subject: [PATCH 9/9] chore: Set verify=false in post request as a workaround --- src/unity_initiator/actions/submit_dag_by_id.py | 4 +++- src/unity_initiator/actions/submit_ogc_process_execution.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/unity_initiator/actions/submit_dag_by_id.py b/src/unity_initiator/actions/submit_dag_by_id.py index 1fcc20a..d8019bc 100644 --- a/src/unity_initiator/actions/submit_dag_by_id.py +++ b/src/unity_initiator/actions/submit_dag_by_id.py @@ -34,7 +34,9 @@ def execute(self): }, "note": "", } - response = httpx.post(url, auth=auth, headers=headers, json=body) + response = httpx.post( + url, auth=auth, headers=headers, json=body, verify=False + ) # nosec if response.status_code in (200, 201): success = True resp = response.json() diff --git a/src/unity_initiator/actions/submit_ogc_process_execution.py b/src/unity_initiator/actions/submit_ogc_process_execution.py index 56df2ce..9b3fcee 100644 --- a/src/unity_initiator/actions/submit_ogc_process_execution.py +++ b/src/unity_initiator/actions/submit_ogc_process_execution.py @@ -30,7 +30,7 @@ def execute(self): "outputs": None, "subscriber": None, } - response = httpx.post(url, headers=headers, json=body) + response = httpx.post(url, headers=headers, json=body, verify=False) # nosec if response.status_code in (200, 201): success = True resp = response.json()