From f1171996dc5fd8d2508e4927f5b8b0885f989422 Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Mon, 12 Aug 2024 14:16:21 -0700
Subject: [PATCH 01/11] testing out X-Ray tracing

---
 .../build_scheduled_task_lambda_package.sh    |  21 +++
 src/unity_initiator/cloud/lambda_handler.py   |  39 +++---
 terraform-unity/initiator/README.md           |   1 +
 terraform-unity/initiator/data.tf             |   2 +-
 terraform-unity/initiator/main.tf             |  21 ++-
 .../triggers/cmr_query/versions.tf            |  23 +--
 .../.terraform.lock.hcl                       |  85 +++++++++++
 .../scheduled_task-instrumented/README.md     |  53 +++++++
 .../scheduled_task-instrumented/data.tf       |   8 ++
 .../lambda_handler.py                         |  31 ++++
 .../scheduled_task-instrumented/locals.tf     |  15 ++
 .../scheduled_task-instrumented/main.tf       | 132 ++++++++++++++++++
 .../scheduled_task-instrumented/variables.tf  |  26 ++++
 .../scheduled_task-instrumented/versions.tf   |   1 +
 14 files changed, 414 insertions(+), 44 deletions(-)
 create mode 100755 scripts/build_scheduled_task_lambda_package.sh
 mode change 100644 => 120000 terraform-unity/triggers/cmr_query/versions.tf
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/.terraform.lock.hcl
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/README.md
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/data.tf
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/locals.tf
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/main.tf
 create mode 100644 terraform-unity/triggers/scheduled_task-instrumented/variables.tf
 create mode 120000 terraform-unity/triggers/scheduled_task-instrumented/versions.tf

diff --git a/scripts/build_scheduled_task_lambda_package.sh b/scripts/build_scheduled_task_lambda_package.sh
new file mode 100755
index 0000000..c3a82e0
--- /dev/null
+++ b/scripts/build_scheduled_task_lambda_package.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+BASE_PATH=$(dirname "${BASH_SOURCE}")
+BASE_PATH=$(cd "${BASE_PATH}/.."; pwd)
+DIST_DIR=${BASE_PATH}/dist
+PKG_DIR=${DIST_DIR}/lambda_packages
+SCHED_TASK_DIR=${BASE_PATH}/terraform-unity/triggers/scheduled_task-instrumented
+
+set -ex
+
+rm -rf $DIST_DIR
+pip install hatch
+hatch clean
+hatch build
+VERSION=$(hatch run python -c 'from importlib.metadata import version; print(version("unity_initiator"))')
+echo "{\"version\": \"$VERSION\"}" > ${DIST_DIR}/version.json
+mkdir -p $PKG_DIR
+pip install -t $PKG_DIR ${DIST_DIR}/unity_initiator-*.whl
+pip install -t $PKG_DIR aws_xray_sdk
+cp ${SCHED_TASK_DIR}/lambda_handler.py $PKG_DIR/
+cd $PKG_DIR
+zip -rq ${DIST_DIR}/scheduled_task-${VERSION}-lambda.zip .
diff --git a/src/unity_initiator/cloud/lambda_handler.py b/src/unity_initiator/cloud/lambda_handler.py
index 87e9fd7..b3e5a72 100644
--- a/src/unity_initiator/cloud/lambda_handler.py
+++ b/src/unity_initiator/cloud/lambda_handler.py
@@ -4,10 +4,15 @@
 from tempfile import mkstemp
 
 import smart_open
+from aws_xray_sdk.core import patch_all, xray_recorder
 
 from ..router import Router
 from ..utils.logger import logger
 
+# initialize the AWS X-Ray SDK
+patch_all()
+
+
 ROUTER = None
 
 
@@ -20,22 +25,24 @@ def lambda_handler_base(event, context):
     # or from a url in ROUTER_CFG_URL env variable.
     global ROUTER
     if ROUTER is None:
-        router_cfg = os.environ.get("ROUTER_CFG", "").strip()
-        router_cfg_url = os.environ.get("ROUTER_CFG_URL", "").strip()
-        if router_cfg == "":
-            if router_cfg_url != "":
-                with smart_open.open(router_cfg_url, "r") as f:
-                    router_cfg = f.read()
-            else:
-                raise RuntimeError(
-                    "No router configuration specified via ROUTER_CFG or ROUTER_CFG_URL env variables."
-                )
-        fd, router_file = mkstemp(prefix="router_", suffix=".yaml", text=True)
-        with os.fdopen(fd, "w") as f:
-            f.write(router_cfg)
-        ROUTER = Router(router_file)
-        os.unlink(router_file)
-    return ROUTER.execute_actions(event["payload"])
+        with xray_recorder.capture("read_router_config"):
+            router_cfg = os.environ.get("ROUTER_CFG", "").strip()
+            router_cfg_url = os.environ.get("ROUTER_CFG_URL", "").strip()
+            if router_cfg == "":
+                if router_cfg_url != "":
+                    with smart_open.open(router_cfg_url, "r") as f:
+                        router_cfg = f.read()
+                else:
+                    raise RuntimeError(
+                        "No router configuration specified via ROUTER_CFG or ROUTER_CFG_URL env variables."
+                    )
+            fd, router_file = mkstemp(prefix="router_", suffix=".yaml", text=True)
+            with os.fdopen(fd, "w") as f:
+                f.write(router_cfg)
+            ROUTER = Router(router_file)
+            os.unlink(router_file)
+    with xray_recorder.capture("execute_actions"):
+        return ROUTER.execute_actions(event["payload"])
 
 
 def lambda_handler_multiple_payloads(event, context):
diff --git a/terraform-unity/initiator/README.md b/terraform-unity/initiator/README.md
index 82e0dc9..536defc 100644
--- a/terraform-unity/initiator/README.md
+++ b/terraform-unity/initiator/README.md
@@ -29,6 +29,7 @@ No modules.
 | [aws_cloudwatch_log_group.initiator_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_iam_policy.initiator_lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.initiator_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.aws_xray_write_only_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_sns_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
diff --git a/terraform-unity/initiator/data.tf b/terraform-unity/initiator/data.tf
index 87302c4..d2dcb3e 100644
--- a/terraform-unity/initiator/data.tf
+++ b/terraform-unity/initiator/data.tf
@@ -5,4 +5,4 @@ data "aws_iam_policy" "mcp_operator_policy" {
 data "local_file" "version" {
   filename   = "${path.module}/../../dist/version.json"
   depends_on = [null_resource.build_lambda_package]
-}
\ No newline at end of file
+}
diff --git a/terraform-unity/initiator/main.tf b/terraform-unity/initiator/main.tf
index bb6fefc..e493224 100644
--- a/terraform-unity/initiator/main.tf
+++ b/terraform-unity/initiator/main.tf
@@ -40,6 +40,11 @@ resource "aws_lambda_function" "initiator_lambda" {
       ROUTER_CFG_URL = "s3://${aws_s3_object.router_config.bucket}/${aws_s3_object.router_config.key}"
     }
   }
+
+  tracing_config {
+    mode = "Active"
+  }
+
   tags = local.tags
 }
 
@@ -116,6 +121,11 @@ resource "aws_iam_role_policy_attachment" "lambda_base_policy_attachment" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
+resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" {
+  role       = aws_iam_role.initiator_lambda_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
+}
+
 resource "aws_ssm_parameter" "initiator_lambda_function_name" {
   name  = "/unity/${var.project}/${var.venue}/od/initiator/lambda-name"
   type  = "String"
@@ -147,11 +157,6 @@ resource "aws_sqs_queue" "initiator_queue" {
   tags = local.tags
 }
 
-resource "aws_sns_topic" "initiator_topic" {
-  name = "${local.function_name}_topic"
-  tags = local.tags
-}
-
 resource "aws_sqs_queue_policy" "initiator_queue_policy" {
   queue_url = aws_sqs_queue.initiator_queue.id
   policy = jsonencode({
@@ -172,6 +177,12 @@ resource "aws_sqs_queue_policy" "initiator_queue_policy" {
   })
 }
 
+resource "aws_sns_topic" "initiator_topic" {
+  name           = "${local.function_name}_topic"
+  tags           = local.tags
+  tracing_config = "Active"
+}
+
 resource "aws_sns_topic_subscription" "initiator_subscription" {
   topic_arn = aws_sns_topic.initiator_topic.arn
   protocol  = "sqs"
diff --git a/terraform-unity/triggers/cmr_query/versions.tf b/terraform-unity/triggers/cmr_query/versions.tf
deleted file mode 100644
index 5e8229b..0000000
--- a/terraform-unity/triggers/cmr_query/versions.tf
+++ /dev/null
@@ -1,22 +0,0 @@
-terraform {
-  required_version = "~> 1.4.6"
-
-  required_providers {
-    archive = {
-      source  = "hashicorp/archive"
-      version = ">=2.4.2"
-    }
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">=5.50.0"
-    }
-    local = {
-      source  = "hashicorp/local"
-      version = ">=2.5.1"
-    }
-    null = {
-      source  = "hashicorp/null"
-      version = ">=3.2.2"
-    }
-  }
-}
diff --git a/terraform-unity/triggers/cmr_query/versions.tf b/terraform-unity/triggers/cmr_query/versions.tf
new file mode 120000
index 0000000..53530d2
--- /dev/null
+++ b/terraform-unity/triggers/cmr_query/versions.tf
@@ -0,0 +1 @@
+../scheduled_task/versions.tf
\ No newline at end of file
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/.terraform.lock.hcl b/terraform-unity/triggers/scheduled_task-instrumented/.terraform.lock.hcl
new file mode 100644
index 0000000..8cbe489
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/.terraform.lock.hcl
@@ -0,0 +1,85 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/archive" {
+  version     = "2.4.2"
+  constraints = ">= 2.4.2"
+  hashes = [
+    "h1:1eOz9vM/55vnQjxk23RhnYga7PZq8n2rGxG+2Vx2s6w=",
+    "zh:08faed7c9f42d82bc3d406d0d9d4971e2d1c2d34eae268ad211b8aca57b7f758",
+    "zh:3564112ed2d097d7e0672378044a69b06642c326f6f1584d81c7cdd32ebf3a08",
+    "zh:53cd9afd223c15828c1916e68cb728d2be1cbccb9545568d6c2b122d0bac5102",
+    "zh:5ae4e41e3a1ce9d40b6458218a85bbde44f21723943982bca4a3b8bb7c103670",
+    "zh:5b65499218b315b96e95c5d3463ea6d7c66245b59461217c99eaa1611891cd2c",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:7f45b35a8330bebd184c2545a41782ff58240ed6ba947274d9881dd5da44b02e",
+    "zh:87e67891033214e55cfead1391d68e6a3bf37993b7607753237e82aa3250bb71",
+    "zh:de3590d14037ad81fc5cedf7cfa44614a92452d7b39676289b704a962050bc5e",
+    "zh:e7e6f2ea567f2dbb3baa81c6203be69f9cd6aeeb01204fd93e3cf181e099b610",
+    "zh:fd24d03c89a7702628c2e5a3c732c0dede56fa75a08da4a1efe17b5f881c88e2",
+    "zh:febf4b7b5f3ff2adff0573ef6361f09b6638105111644bdebc0e4f575373935f",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.51.1"
+  constraints = ">= 5.50.0"
+  hashes = [
+    "h1:KY/uPHIa+bHgMOAqoA2BnjIlIDuFRFwbLjLkf1gbeDk=",
+    "zh:03d524b70ab300d90dc4dccad0c28b18d797b8986722b7a93e40a41500450eaa",
+    "zh:04dbcb7ab52181a784877c409f6c882df34bda686d8c884d511ebd4abf493f0c",
+    "zh:2b068f7838e0f3677829258df05d8b9d73fe6434a1a809f8710956cc1c01ea03",
+    "zh:41a4b1e4adbf7c90015ebff17a719fc08133b8a2c4dcefd2fa281552126e59a8",
+    "zh:48b1adf57f695a72c88c598f99912171ef7067638fd63fb0c6ad3fa397b3f7c3",
+    "zh:5c2fb26ecb83adac90d06dcf5f97edbc944824c2821816b1653e1a2b9d37b3c4",
+    "zh:93df05f53702df829d9b9335e559ad8b313808dbd2fad8b2ff14f176732e693d",
+    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+    "zh:b5da39898602e44551b56e2803a42d92ea7115e35b1792efbf6649da37ef597b",
+    "zh:b7ab7f743f864ed8d479a7cb04fd3ce00c376f867ee5b53c4c1acaef6e286c54",
+    "zh:e7e7b2d8ee486415481a25ac7bdded20bd2897d5dd0790741798f31935b9528d",
+    "zh:e8008e3f5ef560fd9004d1ed1738f0f53e99b0ce961d967e95fc7c02e5954e4e",
+    "zh:f1296f648b8608ffa930b52519b00ed01eebedde9fdaf94205b365536e6c3916",
+    "zh:f8539960fd978a54990740ee984c6f7f743c9c32c7734e2601e92abfe54367e9",
+    "zh:fd182e6e20bb52982752a5d8c4b16887565f413a9d50d9d394d2c06eea8a195e",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/local" {
+  version     = "2.5.1"
+  constraints = ">= 2.5.1"
+  hashes = [
+    "h1:/GAVA/xheGQcbOZEq0qxANOg+KVLCA7Wv8qluxhTjhU=",
+    "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561",
+    "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561",
+    "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5",
+    "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24",
+    "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667",
+    "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8",
+    "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0",
+    "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867",
+    "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4",
+    "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/null" {
+  version     = "3.2.2"
+  constraints = ">= 3.2.2"
+  hashes = [
+    "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=",
+    "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7",
+    "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a",
+    "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3",
+    "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606",
+    "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546",
+    "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539",
+    "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422",
+    "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae",
+    "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1",
+    "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e",
+  ]
+}
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/README.md b/terraform-unity/triggers/scheduled_task-instrumented/README.md
new file mode 100644
index 0000000..f8ff0da
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/README.md
@@ -0,0 +1,53 @@
+# scheduled_task
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_archive"></a> [archive](#requirement\_archive) | >=2.4.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
+| <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |
+| <a name="requirement_null"></a> [null](#requirement\_null) | >=3.2.2 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_archive"></a> [archive](#provider\_archive) | 2.4.2 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.51.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_group.scheduled_task_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_iam_policy.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.scheduled_task_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.lambda_sns_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_lambda_function.scheduled_task_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
+| [aws_scheduler_schedule.run_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule) | resource |
+| [archive_file.lambda_zip_inline](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
+| [aws_iam_policy.mcp_operator_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes |
+| <a name="input_initiator_topic_arn"></a> [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes |
+| <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
+| <a name="input_venue"></a> [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/data.tf b/terraform-unity/triggers/scheduled_task-instrumented/data.tf
new file mode 100644
index 0000000..d8b3d82
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/data.tf
@@ -0,0 +1,8 @@
+data "aws_iam_policy" "mcp_operator_policy" {
+  name = "mcp-tenantOperator-AMI-APIG"
+}
+
+data "local_file" "version" {
+  filename   = "${path.module}/../../../dist/version.json"
+  depends_on = [null_resource.build_lambda_package]
+}
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py b/terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py
new file mode 100644
index 0000000..c86b217
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py
@@ -0,0 +1,31 @@
+import os
+import json
+import boto3
+from aws_xray_sdk.core import patch_all, xray_recorder
+
+
+patch_all()
+
+
+INITIATOR_TOPIC_ARN = os.environ["INITIATOR_TOPIC_ARN"]
+
+def lambda_handler(event, context):
+    print(f"event: {json.dumps(event, indent=2)}")
+    print(f"context: {context}")
+
+    # implement your adaptation-specific trigger code here and submit payloads
+    # to the SNS topic as either a list of payloads or a single payload. Below
+    # is an example of a single payload.
+    # Finally return True if it successful. False otherwise.
+
+    with xray_recorder.capture("publish_url_to_inititator_topic"):
+        client = boto3.client("sns")
+        res = client.publish(
+            TopicArn=INITIATOR_TOPIC_ARN,
+            Subject="Scheduled Task",
+            Message=json.dumps({"payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"})
+        )
+        return {
+            "success": True,
+            "response": res
+        }
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/locals.tf b/terraform-unity/triggers/scheduled_task-instrumented/locals.tf
new file mode 100644
index 0000000..68396e8
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/locals.tf
@@ -0,0 +1,15 @@
+locals {
+  function_name = "${var.project}-${var.venue}-${var.deployment_name}-scheduled_task"
+  tags = {
+    Venue       = "dev"
+    ServiceArea = "cs"
+    Capability  = "trigger"
+    CapVersion  = "0.0.1"
+    Component   = "U-OD"
+    Name        = "${var.project}-${var.venue}-cs-trigger-od"
+    Proj        = var.project
+    CreatedBy   = "cs"
+    Env         = "dev"
+    Stack       = "U-OD"
+  }
+}
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/main.tf b/terraform-unity/triggers/scheduled_task-instrumented/main.tf
new file mode 100644
index 0000000..b6e3c39
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/main.tf
@@ -0,0 +1,132 @@
+resource "null_resource" "build_lambda_package" {
+  triggers = { always_run = timestamp() }
+  provisioner "local-exec" {
+    command = <<EOF
+      set -ex
+      docker run --rm -v ${path.module}/../../..:/var/task mlupin/docker-lambda:python3.9-build ./scripts/build_scheduled_task_lambda_package.sh
+    EOF
+  }
+}
+
+resource "aws_s3_object" "lambda_package" {
+  depends_on = [null_resource.build_lambda_package]
+  bucket     = var.code_bucket
+  key        = "scheduled_task-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  source     = "${path.module}/../../../dist/scheduled_task-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  etag       = filemd5("${path.module}/../../../dist/scheduled_task-${jsondecode(data.local_file.version.content).version}-lambda.zip")
+  tags       = local.tags
+}
+
+resource "aws_iam_role" "scheduled_task_lambda_iam_role" {
+  name = "${local.function_name}_lambda_iam_role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Action = "sts:AssumeRole",
+        Effect = "Allow",
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        },
+      },
+    ],
+  })
+  permissions_boundary = data.aws_iam_policy.mcp_operator_policy.arn
+  tags                 = local.tags
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_sns_policy_attachment" {
+  role       = aws_iam_role.scheduled_task_lambda_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSNSFullAccess"
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_base_policy_attachment" {
+  role       = aws_iam_role.scheduled_task_lambda_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}
+
+resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" {
+  role       = aws_iam_role.scheduled_task_lambda_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
+}
+
+resource "aws_lambda_function" "scheduled_task_lambda" {
+  depends_on    = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.scheduled_task_lambda_log_group]
+  function_name    = local.function_name
+  s3_bucket        = var.code_bucket
+  s3_key           = "scheduled_task-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  handler          = "lambda_handler.lambda_handler"
+  runtime          = "python3.11"
+  role             = aws_iam_role.scheduled_task_lambda_iam_role.arn
+  timeout          = 900
+
+  environment {
+    variables = {
+      INITIATOR_TOPIC_ARN = var.initiator_topic_arn
+    }
+  }
+
+  tracing_config {
+    mode = "Active"
+  }
+
+  tags = local.tags
+}
+
+resource "aws_cloudwatch_log_group" "scheduled_task_lambda_log_group" {
+  name              = "/aws/lambda/${local.function_name}"
+  retention_in_days = 14
+}
+
+resource "aws_iam_role" "scheduler" {
+  name = "${var.project}-${var.venue}-${var.deployment_name}-cron-scheduler-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Principal = {
+          Service = ["scheduler.amazonaws.com"]
+        }
+        Action = "sts:AssumeRole"
+      }
+    ]
+  })
+  permissions_boundary = data.aws_iam_policy.mcp_operator_policy.arn
+  tags                 = local.tags
+}
+
+resource "aws_iam_policy" "scheduler" {
+  name = "${var.project}-${var.venue}-${var.deployment_name}-cron-scheduler-policy"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        # allow scheduler to execute the task
+        Effect = "Allow",
+        Action = [
+          "lambda:InvokeFunction"
+        ]
+        Resource = aws_lambda_function.scheduled_task_lambda.arn
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "scheduler" {
+  policy_arn = aws_iam_policy.scheduler.arn
+  role       = aws_iam_role.scheduler.name
+}
+
+resource "aws_scheduler_schedule" "run_scheduled_task" {
+  name                = "${var.project}-${var.venue}-${var.deployment_name}-run_scheduled_task"
+  schedule_expression = "rate(1 minute)"
+  flexible_time_window {
+    mode = "OFF"
+  }
+  target {
+    arn      = aws_lambda_function.scheduled_task_lambda.arn
+    role_arn = aws_iam_role.scheduler.arn
+  }
+}
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/variables.tf b/terraform-unity/triggers/scheduled_task-instrumented/variables.tf
new file mode 100644
index 0000000..56b95f9
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/variables.tf
@@ -0,0 +1,26 @@
+variable "deployment_name" {
+  description = "The deployment name"
+  type        = string
+}
+
+variable "project" {
+  description = "The unity project its installed into"
+  type        = string
+  default     = "uod"
+}
+
+variable "venue" {
+  description = "The unity venue its installed into"
+  type        = string
+  default     = "dev"
+}
+
+variable "code_bucket" {
+  description = "The S3 bucket where lambda zip files will be stored and accessed"
+  type        = string
+}
+
+variable "initiator_topic_arn" {
+  description = "The ARN of the initiator SNS topic to publish S3 events to"
+  type        = string
+}
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/versions.tf b/terraform-unity/triggers/scheduled_task-instrumented/versions.tf
new file mode 120000
index 0000000..53530d2
--- /dev/null
+++ b/terraform-unity/triggers/scheduled_task-instrumented/versions.tf
@@ -0,0 +1 @@
+../scheduled_task/versions.tf
\ No newline at end of file

From f317dee61630aa001fe4b550031b819759798487 Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Mon, 12 Aug 2024 14:50:08 -0700
Subject: [PATCH 02/11] use dashes instead of underscores, run pre-commit and
 update README.md

---
 README.md                                      | 18 +++++++++---------
 .../.terraform.lock.hcl                        |  0
 .../{cmr_query => cmr-query}/README.md         |  0
 .../triggers/{cmr_query => cmr-query}/data.tf  |  0
 .../{cmr_query => cmr-query}/lambda_handler.py |  0
 .../{cmr_query => cmr-query}/locals.tf         |  0
 .../triggers/{cmr_query => cmr-query}/main.tf  |  0
 .../{cmr_query => cmr-query}/variables.tf      |  0
 terraform-unity/triggers/cmr-query/versions.tf |  1 +
 terraform-unity/triggers/cmr_query/versions.tf |  1 -
 .../.terraform.lock.hcl                        |  0
 .../README.md                                  | 11 ++++++++---
 .../data.tf                                    |  0
 .../lambda_handler.py                          | 16 +++++++++-------
 .../locals.tf                                  |  0
 .../main.tf                                    | 14 +++++++-------
 .../variables.tf                               |  0
 .../scheduled-task-instrumented/versions.tf    |  1 +
 .../scheduled_task-instrumented/versions.tf    |  1 -
 19 files changed, 35 insertions(+), 28 deletions(-)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/.terraform.lock.hcl (100%)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/README.md (100%)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/data.tf (100%)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/lambda_handler.py (100%)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/locals.tf (100%)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/main.tf (100%)
 rename terraform-unity/triggers/{cmr_query => cmr-query}/variables.tf (100%)
 create mode 120000 terraform-unity/triggers/cmr-query/versions.tf
 delete mode 120000 terraform-unity/triggers/cmr_query/versions.tf
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/.terraform.lock.hcl (100%)
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/README.md (76%)
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/data.tf (100%)
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/lambda_handler.py (74%)
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/locals.tf (100%)
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/main.tf (91%)
 rename terraform-unity/triggers/{scheduled_task-instrumented => scheduled-task-instrumented}/variables.tf (100%)
 create mode 120000 terraform-unity/triggers/scheduled-task-instrumented/versions.tf
 delete mode 120000 terraform-unity/triggers/scheduled_task-instrumented/versions.tf

diff --git a/README.md b/README.md
index b1e4b2d..3e09a81 100644
--- a/README.md
+++ b/README.md
@@ -280,16 +280,16 @@ This guide provides a quick way to get started with our project. Please see our
 
 #### Deploying an Example Evaluator (SNS topic->SQS queue->Lambda)
 
-1. Change directory to the location of the sns_sqs_lambda evaluator terraform:
+1. Change directory to the location of the sns-sqs-lambda evaluator terraform:
 
    ```
-   cp -rp sns_sqs_lambda sns_sqs_lambda-nisar_tlm
+   cp -rp sns-sqs-lambda sns-sqs-lambda-nisar-tlm
    ```
 
 1. Change directory into the NISAR TLM evaluator terraform:
 
    ```
-   cd sns_sqs_lambda-nisar_tlm/
+   cd sns-sqs-lambda-nisar-tlm/
    ```
 
 1. Set the name of the evaluator to our NISAR example:
@@ -322,10 +322,10 @@ This guide provides a quick way to get started with our project. Please see our
 
 #### Deploying an S3 Event Notification Trigger
 
-1. Change directory to the location of the s3_bucket_notification trigger terraform:
+1. Change directory to the location of the s3-bucket-notification trigger terraform:
 
    ```
-   cd ../../triggers/s3_bucket_notification/
+   cd ../../triggers/s3-bucket-notification/
    ```
 
 1. You will need an S3 bucket to configure event notification on. Create one or reuse an existing one (could be the same one in the previous steps) and set an environment variable for it:
@@ -382,10 +382,10 @@ This guide provides a quick way to get started with our project. Please see our
 
 #### Deploying an EventBridge Scheduler Trigger
 
-1. Change directory to the location of the s3_bucket_notification trigger terraform:
+1. Change directory to the location of the scheduled-task trigger terraform:
 
    ```
-   cd ../scheduled_task/
+   cd ../scheduled-task/
    ```
 
 1. Note the implementation of the trigger lambda code. It currently hard codes a payload URL however in a real implementation, code would be written to query for new files from some REST API, database, etc. Here we simulate that and simply return a NISAR TLM file:
@@ -416,10 +416,10 @@ This guide provides a quick way to get started with our project. Please see our
 
 #### Deploying an EventBridge Scheduler Trigger for Periodic CMR Queries
 
-1. Change directory to the location of the s3_bucket_notification trigger terraform:
+1. Change directory to the location of the cmr-query trigger terraform:
 
    ```
-   cd ../cmr_query/
+   cd ../cmr-query/
    ```
 
 1. Note the implementation of the trigger lambda code. It will query CMR for granules for a particular collection within a timeframe, query its dynamodb table if they already exist, and if not, submit them as payload URLs to the initiator SNS topic and save them into the dynamodb table:
diff --git a/terraform-unity/triggers/cmr_query/.terraform.lock.hcl b/terraform-unity/triggers/cmr-query/.terraform.lock.hcl
similarity index 100%
rename from terraform-unity/triggers/cmr_query/.terraform.lock.hcl
rename to terraform-unity/triggers/cmr-query/.terraform.lock.hcl
diff --git a/terraform-unity/triggers/cmr_query/README.md b/terraform-unity/triggers/cmr-query/README.md
similarity index 100%
rename from terraform-unity/triggers/cmr_query/README.md
rename to terraform-unity/triggers/cmr-query/README.md
diff --git a/terraform-unity/triggers/cmr_query/data.tf b/terraform-unity/triggers/cmr-query/data.tf
similarity index 100%
rename from terraform-unity/triggers/cmr_query/data.tf
rename to terraform-unity/triggers/cmr-query/data.tf
diff --git a/terraform-unity/triggers/cmr_query/lambda_handler.py b/terraform-unity/triggers/cmr-query/lambda_handler.py
similarity index 100%
rename from terraform-unity/triggers/cmr_query/lambda_handler.py
rename to terraform-unity/triggers/cmr-query/lambda_handler.py
diff --git a/terraform-unity/triggers/cmr_query/locals.tf b/terraform-unity/triggers/cmr-query/locals.tf
similarity index 100%
rename from terraform-unity/triggers/cmr_query/locals.tf
rename to terraform-unity/triggers/cmr-query/locals.tf
diff --git a/terraform-unity/triggers/cmr_query/main.tf b/terraform-unity/triggers/cmr-query/main.tf
similarity index 100%
rename from terraform-unity/triggers/cmr_query/main.tf
rename to terraform-unity/triggers/cmr-query/main.tf
diff --git a/terraform-unity/triggers/cmr_query/variables.tf b/terraform-unity/triggers/cmr-query/variables.tf
similarity index 100%
rename from terraform-unity/triggers/cmr_query/variables.tf
rename to terraform-unity/triggers/cmr-query/variables.tf
diff --git a/terraform-unity/triggers/cmr-query/versions.tf b/terraform-unity/triggers/cmr-query/versions.tf
new file mode 120000
index 0000000..ba06b3f
--- /dev/null
+++ b/terraform-unity/triggers/cmr-query/versions.tf
@@ -0,0 +1 @@
+../scheduled-task/versions.tf
\ No newline at end of file
diff --git a/terraform-unity/triggers/cmr_query/versions.tf b/terraform-unity/triggers/cmr_query/versions.tf
deleted file mode 120000
index 53530d2..0000000
--- a/terraform-unity/triggers/cmr_query/versions.tf
+++ /dev/null
@@ -1 +0,0 @@
-../scheduled_task/versions.tf
\ No newline at end of file
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/.terraform.lock.hcl b/terraform-unity/triggers/scheduled-task-instrumented/.terraform.lock.hcl
similarity index 100%
rename from terraform-unity/triggers/scheduled_task-instrumented/.terraform.lock.hcl
rename to terraform-unity/triggers/scheduled-task-instrumented/.terraform.lock.hcl
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/README.md b/terraform-unity/triggers/scheduled-task-instrumented/README.md
similarity index 76%
rename from terraform-unity/triggers/scheduled_task-instrumented/README.md
rename to terraform-unity/triggers/scheduled-task-instrumented/README.md
index f8ff0da..a1a2d80 100644
--- a/terraform-unity/triggers/scheduled_task-instrumented/README.md
+++ b/terraform-unity/triggers/scheduled-task-instrumented/README.md
@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.8.2 |
 | <a name="requirement_archive"></a> [archive](#requirement\_archive) | >=2.4.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |
@@ -15,8 +15,9 @@
 
 | Name | Version |
 |------|---------|
-| <a name="provider_archive"></a> [archive](#provider\_archive) | 2.4.2 |
 | <a name="provider_aws"></a> [aws](#provider\_aws) | 5.51.1 |
+| <a name="provider_local"></a> [local](#provider\_local) | 2.5.1 |
+| <a name="provider_null"></a> [null](#provider\_null) | 3.2.2 |
 
 ## Modules
 
@@ -30,18 +31,22 @@ No modules.
 | [aws_iam_policy.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.scheduled_task_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.aws_xray_write_only_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_sns_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_lambda_function.scheduled_task_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
+| [aws_s3_object.lambda_package](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_scheduler_schedule.run_scheduled_task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule) | resource |
-| [archive_file.lambda_zip_inline](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
+| [null_resource.build_lambda_package](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [aws_iam_policy.mcp_operator_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source |
+| [local_file.version](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_code_bucket"></a> [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes |
 | <a name="input_initiator_topic_arn"></a> [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes |
 | <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/data.tf b/terraform-unity/triggers/scheduled-task-instrumented/data.tf
similarity index 100%
rename from terraform-unity/triggers/scheduled_task-instrumented/data.tf
rename to terraform-unity/triggers/scheduled-task-instrumented/data.tf
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py b/terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py
similarity index 74%
rename from terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py
rename to terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py
index c86b217..5edc54e 100644
--- a/terraform-unity/triggers/scheduled_task-instrumented/lambda_handler.py
+++ b/terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py
@@ -1,14 +1,15 @@
-import os
 import json
+import os
+
 import boto3
 from aws_xray_sdk.core import patch_all, xray_recorder
 
-
 patch_all()
 
 
 INITIATOR_TOPIC_ARN = os.environ["INITIATOR_TOPIC_ARN"]
 
+
 def lambda_handler(event, context):
     print(f"event: {json.dumps(event, indent=2)}")
     print(f"context: {context}")
@@ -23,9 +24,10 @@ def lambda_handler(event, context):
         res = client.publish(
             TopicArn=INITIATOR_TOPIC_ARN,
             Subject="Scheduled Task",
-            Message=json.dumps({"payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"})
+            Message=json.dumps(
+                {
+                    "payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"
+                }
+            ),
         )
-        return {
-            "success": True,
-            "response": res
-        }
+        return {"success": True, "response": res}
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/locals.tf b/terraform-unity/triggers/scheduled-task-instrumented/locals.tf
similarity index 100%
rename from terraform-unity/triggers/scheduled_task-instrumented/locals.tf
rename to terraform-unity/triggers/scheduled-task-instrumented/locals.tf
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/main.tf b/terraform-unity/triggers/scheduled-task-instrumented/main.tf
similarity index 91%
rename from terraform-unity/triggers/scheduled_task-instrumented/main.tf
rename to terraform-unity/triggers/scheduled-task-instrumented/main.tf
index b6e3c39..aa6aad2 100644
--- a/terraform-unity/triggers/scheduled_task-instrumented/main.tf
+++ b/terraform-unity/triggers/scheduled-task-instrumented/main.tf
@@ -53,13 +53,13 @@ resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" {
 
 resource "aws_lambda_function" "scheduled_task_lambda" {
   depends_on    = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.scheduled_task_lambda_log_group]
-  function_name    = local.function_name
-  s3_bucket        = var.code_bucket
-  s3_key           = "scheduled_task-${jsondecode(data.local_file.version.content).version}-lambda.zip"
-  handler          = "lambda_handler.lambda_handler"
-  runtime          = "python3.11"
-  role             = aws_iam_role.scheduled_task_lambda_iam_role.arn
-  timeout          = 900
+  function_name = local.function_name
+  s3_bucket     = var.code_bucket
+  s3_key        = "scheduled_task-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  handler       = "lambda_handler.lambda_handler"
+  runtime       = "python3.11"
+  role          = aws_iam_role.scheduled_task_lambda_iam_role.arn
+  timeout       = 900
 
   environment {
     variables = {
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/variables.tf b/terraform-unity/triggers/scheduled-task-instrumented/variables.tf
similarity index 100%
rename from terraform-unity/triggers/scheduled_task-instrumented/variables.tf
rename to terraform-unity/triggers/scheduled-task-instrumented/variables.tf
diff --git a/terraform-unity/triggers/scheduled-task-instrumented/versions.tf b/terraform-unity/triggers/scheduled-task-instrumented/versions.tf
new file mode 120000
index 0000000..ba06b3f
--- /dev/null
+++ b/terraform-unity/triggers/scheduled-task-instrumented/versions.tf
@@ -0,0 +1 @@
+../scheduled-task/versions.tf
\ No newline at end of file
diff --git a/terraform-unity/triggers/scheduled_task-instrumented/versions.tf b/terraform-unity/triggers/scheduled_task-instrumented/versions.tf
deleted file mode 120000
index 53530d2..0000000
--- a/terraform-unity/triggers/scheduled_task-instrumented/versions.tf
+++ /dev/null
@@ -1 +0,0 @@
-../scheduled_task/versions.tf
\ No newline at end of file

From 924c7cbb24954929498b77d0d5af81b4c3a6568b Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Mon, 12 Aug 2024 22:19:28 -0700
Subject: [PATCH 03/11] fix README.md instructions for demo; fix paths

---
 README.md                                     | 80 +++++++++++++++----
 scripts/build_cmr_query_lambda_package.sh     |  2 +-
 .../build_scheduled_task_lambda_package.sh    |  2 +-
 .../lambda_handler.py                         |  8 +-
 .../triggers/scheduled-task/data.tf           |  2 +-
 5 files changed, 71 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 3e09a81..e5c6d52 100644
--- a/README.md
+++ b/README.md
@@ -230,7 +230,13 @@ This guide provides a quick way to get started with our project. Please see our
    cd unity-initiator/terraform-unity/initiator/
    ```
 
-1. Copy a sample router configuration YAML file to use for deployment and update the AWS region and AWS account ID to match your AWS environment. We will be using the NISAR TLM test case for this demo so we also rename the SNS topic ARN for it accordingly:
+1. You will need an S3 bucket for terraform to stage the router Lambda zip file and router configuration YAML file during deployment. Create one or reuse an existing one and set an environment variable for it:
+
+   ```
+   export CODE_BUCKET=<some S3 bucket name>
+   ```
+
+1. Copy a sample router configuration YAML file to use for deployment and update the AWS region and AWS account ID to match your AWS environment. We will be using the NISAR TLM test case for this demo so we also rename the SNS topic ARN for it accordingly. We then upload the router configuration file:
 
    ```
    cp ../../tests/resources/test_router.yaml .
@@ -239,18 +245,7 @@ This guide provides a quick way to get started with our project. Please see our
    sed -i "s/hilo-hawaii-1/${AWS_REGION}/g" test_router.yaml
    sed -i "s/123456789012:eval_nisar_ingest/${AWS_ACCOUNT_ID}:uod-dev-eval_nisar_ingest-evaluator_topic/g" test_router.yaml
    sed -i "s/123456789012:eval_airs_ingest/${AWS_ACCOUNT_ID}:uod-dev-eval_airs_ingest-evaluator_topic/g" test_router.yaml
-   ```
-
-1. You will need an S3 bucket for terraform to stage the router Lambda zip file during deployment. Create one or reuse an existing one and set an environment variable for it:
-
-   ```
-   export CODE_BUCKET=<some S3 bucket name>
-   ```
-
-1. You will need an S3 bucket to store the router configuration YAML file. Create one or reuse an existing one (could be the same one in the previous step) and set an environment variable for it:
-
-   ```
-   export CONFIG_BUCKET=<some S3 bucket name>
+   aws s3 cp test_router.yaml s3://${CODE_BUCKET}/test_router.yaml
    ```
 
 1. Set a project name:
@@ -271,16 +266,27 @@ This guide provides a quick way to get started with our project. Please see our
    terraform apply \
      --var project=${PROJECT} \
      --var code_bucket=${CODE_BUCKET} \
-     --var config_bucket=${CONFIG_BUCKET} \
-     --var router_config=test_router.yaml \
+     --var router_config=s3://${CODE_BUCKET}/test_router.yaml \
      -auto-approve
    ```
 
    **Take note of the `initiator_topic_arn` that is output by terraform. It will be used when setting up any triggers.**
 
-#### Deploying an Example Evaluator (SNS topic->SQS queue->Lambda)
+#### Deploying Example Evaluators (SNS topic->SQS queue->Lambda)
+
+In this demo we will deploy 2 evaluators:
+
+1. `eval_nisar_ingest` - evaluate ingestion of NISAR telemetry files deposited into the ISL bucket
+
+1. `eval_airs_ingest` - evaluate ingestion of AIRS RetStd files returned by a periodic CMR query
+
+##### Evaluator Deployment for NISAR TLM (via staged data to the ISL)
+1. Change directory to the location of the evaluators terraform:
+   ```
+   cd ../evaluators
+   ```
 
-1. Change directory to the location of the sns-sqs-lambda evaluator terraform:
+1. Make a copy of the `sns_sqs_lambda` directory for the NISAR TLM evaluator:
 
    ```
    cp -rp sns-sqs-lambda sns-sqs-lambda-nisar-tlm
@@ -320,6 +326,46 @@ This guide provides a quick way to get started with our project. Please see our
 
    **Take note of the `evaluator_topic_arn` that is output by terraform. It should match the topic ARN in the test_router.yaml file you used during the initiator deployment. If they match then the router Lambda is now able to submit payloads to this evaluator SNS topic.**
 
+##### Evaluator Deployment for AIRS RetStd (via scheduled CMR query)
+1. Change directory to the location of the evaluators terraform:
+   ```
+   cd ..
+   ```
+
+1. Make a copy of the `sns_sqs_lambda` directory for the AIRS RetStd evaluator:
+   ```
+   cp -rp sns-sqs-lambda sns-sqs-lambda-airs-retstd
+   ```
+
+1. Change directory into the AIRS RetStd evaluator terraform:
+   ```
+   cd sns-sqs-lambda-airs-retstd/
+   ```
+
+1. Set the name of the evaluator to our AIRS example:
+   ```
+   export EVALUATOR_NAME=eval_airs_ingest
+   ```
+
+1. Note the implementation of the evaluator code. It currently doesn't do any real evaluation but simply returns that evaluation was successful:
+   ```
+   cat data.tf
+   ```
+
+1. Initialize terraform:
+   ```
+   terraform init
+   ```
+
+1. Run terraform apply:
+   ```
+   terraform apply \
+     --var evaluator_name=${EVALUATOR_NAME} \
+     -auto-approve
+   ```
+
+   **Take note of the `evaluator_topic_arn` that is output by terraform. It should match the respective topic ARN in the test_router.yaml file you used during the initiator deployment. If they match then the router Lambda is now able to submit payloads to this evaluator SNS topic.**
+
 #### Deploying an S3 Event Notification Trigger
 
 1. Change directory to the location of the s3-bucket-notification trigger terraform:
diff --git a/scripts/build_cmr_query_lambda_package.sh b/scripts/build_cmr_query_lambda_package.sh
index 6c1af8e..3c2a2e2 100755
--- a/scripts/build_cmr_query_lambda_package.sh
+++ b/scripts/build_cmr_query_lambda_package.sh
@@ -3,7 +3,7 @@ BASE_PATH=$(dirname "${BASH_SOURCE}")
 BASE_PATH=$(cd "${BASE_PATH}/.."; pwd)
 DIST_DIR=${BASE_PATH}/dist
 PKG_DIR=${DIST_DIR}/lambda_packages
-CMR_QUERY_DIR=${BASE_PATH}/terraform-unity/triggers/cmr_query
+CMR_QUERY_DIR=${BASE_PATH}/terraform-unity/triggers/cmr-query
 
 set -ex
 
diff --git a/scripts/build_scheduled_task_lambda_package.sh b/scripts/build_scheduled_task_lambda_package.sh
index c3a82e0..d757cc4 100755
--- a/scripts/build_scheduled_task_lambda_package.sh
+++ b/scripts/build_scheduled_task_lambda_package.sh
@@ -3,7 +3,7 @@ BASE_PATH=$(dirname "${BASH_SOURCE}")
 BASE_PATH=$(cd "${BASE_PATH}/.."; pwd)
 DIST_DIR=${BASE_PATH}/dist
 PKG_DIR=${DIST_DIR}/lambda_packages
-SCHED_TASK_DIR=${BASE_PATH}/terraform-unity/triggers/scheduled_task-instrumented
+SCHED_TASK_DIR=${BASE_PATH}/terraform-unity/triggers/scheduled-task-instrumented
 
 set -ex
 
diff --git a/terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py b/terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py
index 5edc54e..383c328 100644
--- a/terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py
+++ b/terraform-unity/triggers/scheduled-task-instrumented/lambda_handler.py
@@ -25,9 +25,11 @@ def lambda_handler(event, context):
             TopicArn=INITIATOR_TOPIC_ARN,
             Subject="Scheduled Task",
             Message=json.dumps(
-                {
-                    "payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"
-                }
+                [
+                    {
+                        "payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"
+                    }
+                ]
             ),
         )
         return {"success": True, "response": res}
diff --git a/terraform-unity/triggers/scheduled-task/data.tf b/terraform-unity/triggers/scheduled-task/data.tf
index c9716b9..9cefe5d 100644
--- a/terraform-unity/triggers/scheduled-task/data.tf
+++ b/terraform-unity/triggers/scheduled-task/data.tf
@@ -26,7 +26,7 @@ def lambda_handler(event, context):
     res = client.publish(
         TopicArn=INITIATOR_TOPIC_ARN,
         Subject="Scheduled Task",
-        Message=json.dumps({"payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"})
+        Message=json.dumps([{"payload": "s3://bucket/prefix/NISAR_S198_PA_PA11_M00_P00922_R00_C01_G00_2024_010_17_57_57_714280000.vc25"}])
     )
     return {
         "success": True,

From 0682019a31afb4bddf00ecafe9596af4de3a7ae8 Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Tue, 20 Aug 2024 13:16:01 -0700
Subject: [PATCH 04/11] instrument cmr_query

---
 scripts/build_cmr_query_lambda_package.sh     |  1 +
 terraform-unity/triggers/cmr-query/README.md  |  3 +-
 .../triggers/cmr-query/lambda_handler.py      | 49 ++++++++++++-------
 terraform-unity/triggers/cmr-query/main.tf    | 10 ++++
 .../triggers/cmr-query/variables.tf           |  2 +-
 5 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/scripts/build_cmr_query_lambda_package.sh b/scripts/build_cmr_query_lambda_package.sh
index 3c2a2e2..e2cfbf6 100755
--- a/scripts/build_cmr_query_lambda_package.sh
+++ b/scripts/build_cmr_query_lambda_package.sh
@@ -15,6 +15,7 @@ VERSION=$(hatch run python -c 'from importlib.metadata import version; print(ver
 echo "{\"version\": \"$VERSION\"}" > ${DIST_DIR}/version.json
 mkdir -p $PKG_DIR
 pip install -t $PKG_DIR ${DIST_DIR}/unity_initiator-*.whl
+pip install -t $PKG_DIR aws_xray_sdk
 pip install -t $PKG_DIR python_cmr
 cp ${CMR_QUERY_DIR}/lambda_handler.py $PKG_DIR/
 cd $PKG_DIR
diff --git a/terraform-unity/triggers/cmr-query/README.md b/terraform-unity/triggers/cmr-query/README.md
index 61745ca..43c20ca 100644
--- a/terraform-unity/triggers/cmr-query/README.md
+++ b/terraform-unity/triggers/cmr-query/README.md
@@ -33,6 +33,7 @@ No modules.
 | [aws_iam_policy.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.cmr_query_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.aws_xray_write_only_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_dynamodb_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_sns_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
@@ -55,7 +56,7 @@ No modules.
 | <a name="input_initiator_topic_arn"></a> [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes |
 | <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
 | <a name="input_provider_id"></a> [provider\_id](#input\_provider\_id) | The short name for the data provider: https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#granule-search-by-parameters | `string` | n/a | yes |
-| <a name="input_schedule_expression"></a> [schedule\_expression](#input\_schedule\_expression) | The schedule expression to use for executing the CMR query lambda: https://docs.aws.amazon.com/scheduler/latest/UserGuide/schedule-types.html | `string` | `"rate(1 minute)"` | no |
+| <a name="input_schedule_expression"></a> [schedule\_expression](#input\_schedule\_expression) | The schedule expression to use for executing the CMR query lambda: https://docs.aws.amazon.com/scheduler/latest/UserGuide/schedule-types.html | `string` | `"rate(5 minutes)"` | no |
 | <a name="input_seconds_back"></a> [seconds\_back](#input\_seconds\_back) | Number of seconds back from the current time. Used to create a start and end timerange for a temporal search on granules in the data collection. | `number` | n/a | yes |
 | <a name="input_venue"></a> [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no |
 
diff --git a/terraform-unity/triggers/cmr-query/lambda_handler.py b/terraform-unity/triggers/cmr-query/lambda_handler.py
index 49a60e4..a378eb2 100644
--- a/terraform-unity/triggers/cmr-query/lambda_handler.py
+++ b/terraform-unity/triggers/cmr-query/lambda_handler.py
@@ -3,26 +3,31 @@
 from datetime import datetime, timedelta
 
 import boto3
+from aws_xray_sdk.core import patch_all, xray_recorder
 from cmr import GranuleQuery
 
 from unity_initiator.utils.logger import logger
 
+patch_all()
+
 INITIATOR_TOPIC_ARN = os.environ["INITIATOR_TOPIC_ARN"]
 DYNAMODB_TABLE_NAME = os.environ["DYNAMODB_TABLE_NAME"]
 
 
 def submit_urls_and_bookkeep(sns_client, urls_to_send, table, granules_to_save):
     # batch submit urls to initiator topic
-    res = sns_client.publish(
-        TopicArn=INITIATOR_TOPIC_ARN,
-        Subject="Scheduled Task",
-        Message=json.dumps([{"payload": i} for i in urls_to_send]),
-    )
+    with xray_recorder.capture("publish_url_to_initiator_topic"):
+        res = sns_client.publish(
+            TopicArn=INITIATOR_TOPIC_ARN,
+            Subject="Scheduled Task",
+            Message=json.dumps([{"payload": i} for i in urls_to_send]),
+        )
 
     # batch save submitted granules to table so they are not resubmitted in the future
-    with table.batch_writer() as writer:
-        for granule in granules_to_save:
-            writer.put_item(Item=granule)
+    with xray_recorder.capture("update_url_in_dynamodb_table"):
+        with table.batch_writer() as writer:
+            for granule in granules_to_save:
+                writer.put_item(Item=granule)
 
     return res
 
@@ -38,13 +43,14 @@ def lambda_handler(event, context):
     # create table (or get if already exists) that will be used to track
     # granules that have already been sumbmitted to the initiator
     if DYNAMODB_TABLE_NAME not in db_client.list_tables()["TableNames"]:
-        db_client.create_table(
-            TableName=DYNAMODB_TABLE_NAME,
-            KeySchema=[{"AttributeName": "title", "KeyType": "HASH"}],
-            AttributeDefinitions=[{"AttributeName": "title", "AttributeType": "S"}],
-            ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
-        )
-        logger.info("Created table %s.", DYNAMODB_TABLE_NAME)
+        with xray_recorder.capture("create_dynamodb_table"):
+            db_client.create_table(
+                TableName=DYNAMODB_TABLE_NAME,
+                KeySchema=[{"AttributeName": "title", "KeyType": "HASH"}],
+                AttributeDefinitions=[{"AttributeName": "title", "AttributeType": "S"}],
+                ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+            )
+            logger.info("Created table %s.", DYNAMODB_TABLE_NAME)
     table = boto3.resource("dynamodb").Table(DYNAMODB_TABLE_NAME)
 
     # check required params
@@ -62,10 +68,15 @@ def lambda_handler(event, context):
     logger.info("end_time: %s", end_time)
 
     # query CMR
-    api = GranuleQuery().provider(event["provider_id"]).concept_id(event["concept_id"])
-    api.temporal(start_time.isoformat("T"), end_time.isoformat("T"))
-    hits_count = api.hits()
-    logger.info("total hits: %s", hits_count)
+    with xray_recorder.capture("query_cmr"):
+        api = (
+            GranuleQuery()
+            .provider(event["provider_id"])
+            .concept_id(event["concept_id"])
+        )
+        api.temporal(start_time.isoformat("T"), end_time.isoformat("T"))
+        hits_count = api.hits()
+        logger.info("total hits: %s", hits_count)
 
     # loop over granules and collect ones that haven't been submitted to the
     # initiator yet
diff --git a/terraform-unity/triggers/cmr-query/main.tf b/terraform-unity/triggers/cmr-query/main.tf
index 9538d5a..95019b8 100644
--- a/terraform-unity/triggers/cmr-query/main.tf
+++ b/terraform-unity/triggers/cmr-query/main.tf
@@ -107,6 +107,11 @@ resource "aws_iam_role_policy_attachment" "lambda_dynamodb_policy_attachment" {
   policy_arn = aws_iam_policy.dynamodb_crud_policy.arn
 }
 
+resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" {
+  role       = aws_iam_role.cmr_query_lambda_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
+}
+
 resource "aws_lambda_function" "cmr_query_lambda" {
   depends_on    = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.cmr_query_lambda_log_group]
   function_name = local.function_name
@@ -123,6 +128,11 @@ resource "aws_lambda_function" "cmr_query_lambda" {
       DYNAMODB_TABLE_NAME = aws_dynamodb_table.cmr_table.name
     }
   }
+
+  tracing_config {
+    mode = "Active"
+  }
+
   tags = local.tags
 }
 
diff --git a/terraform-unity/triggers/cmr-query/variables.tf b/terraform-unity/triggers/cmr-query/variables.tf
index 1736fb1..015d993 100644
--- a/terraform-unity/triggers/cmr-query/variables.tf
+++ b/terraform-unity/triggers/cmr-query/variables.tf
@@ -23,7 +23,7 @@ variable "initiator_topic_arn" {
 variable "schedule_expression" {
   description = "The schedule expression to use for executing the CMR query lambda: https://docs.aws.amazon.com/scheduler/latest/UserGuide/schedule-types.html"
   type        = string
-  default     = "rate(1 minute)"
+  default     = "rate(5 minutes)"
 }
 
 variable "provider_id" {

From 71cf92dcbe295b2f2919bc53a06b6bd9a95fcfba Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Tue, 20 Aug 2024 15:42:04 -0700
Subject: [PATCH 05/11] instrument tracing in evaluator; add payload as
 annotation

---
 src/unity_initiator/cloud/lambda_handler.py   |  3 +-
 .../evaluators/sns-sqs-lambda/README.md       |  1 +
 .../evaluators/sns-sqs-lambda/data.tf         | 50 +++++++++++++++++--
 .../evaluators/sns-sqs-lambda/main.tf         |  9 ++++
 4 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/src/unity_initiator/cloud/lambda_handler.py b/src/unity_initiator/cloud/lambda_handler.py
index b3e5a72..1bb75ba 100644
--- a/src/unity_initiator/cloud/lambda_handler.py
+++ b/src/unity_initiator/cloud/lambda_handler.py
@@ -41,7 +41,8 @@ def lambda_handler_base(event, context):
                 f.write(router_cfg)
             ROUTER = Router(router_file)
             os.unlink(router_file)
-    with xray_recorder.capture("execute_actions"):
+    with xray_recorder.capture("execute_actions") as subsegment:
+        subsegment.put_annotation("payload", event["payload"])
         return ROUTER.execute_actions(event["payload"])
 
 
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/README.md b/terraform-unity/evaluators/sns-sqs-lambda/README.md
index da9e8f8..a3c1339 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/README.md
+++ b/terraform-unity/evaluators/sns-sqs-lambda/README.md
@@ -29,6 +29,7 @@ No modules.
 | [aws_cloudwatch_log_group.evaluator_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_iam_policy.evaluator_lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.evaluator_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.aws_xray_write_only_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.lambda_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_lambda_event_source_mapping.evaluator_queue_event_source_mapping](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource |
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/data.tf b/terraform-unity/evaluators/sns-sqs-lambda/data.tf
index 81c9952..28995b4 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/data.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/data.tf
@@ -11,12 +11,56 @@ data "archive_file" "evaluator_lambda_artifact" {
   source {
     filename = "lambda_function.py"
     content  = <<CODE
+import time
+import os
+import binascii
+import json
+
+import boto3
+
+
+def generate_segment_id():
+    return binascii.b2a_hex(os.urandom(8)).decode()
+
+def generate_trace_id(start_time):
+    return "1-{}-{}".format(hex(int(start_time))[2:], binascii.hexlify(os.urandom(12)).decode('utf-8'))
+
 def lambda_handler(event, context):
-    print(f"event: {event}")
+    print(f"event: {json.dumps(event, indent=2, sort_keys=True)}")
     print(f"context: {context}")
 
-    # implement your adaptation-specific evaluator code here and return
-    # True if it successfully evaluates. False otherwise.
+    client = boto3.client("xray")
+
+    for rec in event.get("Records", []):
+
+        # get trace info from AWSTraceHeader if set otherwise set them by default
+        start_time = time.time()
+        aws_trace_header = rec.get("attributes", {}).get("AWSTraceHeader", None)
+        if aws_trace_header is None:
+            trace_id = generate_trace_id(start_time)
+            segment_id = generate_segment_id()
+        else:
+            d = { k:v for k,v in [i.split("=") for i in d.split(";")]}
+            trace_id = d.get("Root", generate_trace_id(start_time))
+            segment_id = d.get("Parent", generate_segment_id())
+
+        # Implement your adaptation-specific evaluator code here and return
+        # True if it successfully evaluates. False otherwise.
+
+        # write trace
+        client.put_trace_segments(
+            TraceSegmentDocuments=[
+                json.dumps(
+                    {
+                        "name": context.function_name,
+                        "id": segment_id,
+                        "start_time": start_time,
+                        "trace_id": trace_id,
+                        "end_time": time.time(),
+                    }
+                )
+            ]
+        )
 
     return { "success": True }
 CODE
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/main.tf b/terraform-unity/evaluators/sns-sqs-lambda/main.tf
index a2dad9c..e406ec7 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/main.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/main.tf
@@ -16,6 +16,10 @@ resource "aws_lambda_function" "evaluator_lambda" {
   filename         = "${path.root}/.archive_files/${var.evaluator_name}-evaluator_lambda.zip"
   source_code_hash = data.archive_file.evaluator_lambda_artifact.output_base64sha256
   tags             = local.tags
+
+  tracing_config {
+    mode = "Active"
+  }
 }
 
 resource "aws_cloudwatch_log_group" "evaluator_lambda_log_group" {
@@ -85,6 +89,11 @@ resource "aws_iam_role_policy_attachment" "lambda_base_policy_attachment" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
+resource "aws_iam_role_policy_attachment" "aws_xray_write_only_access" {
+  role       = aws_iam_role.evaluator_lambda_iam_role.name
+  policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
+}
+
 resource "aws_ssm_parameter" "evaluator_lambda_function_name" {
   name  = "/unity/${var.project}/${var.venue}/od/evaluator/${var.evaluator_name}"
   type  = "String"

From 94f71d1bdc073cfdd23f5ebe36472f01f828f01b Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Tue, 20 Aug 2024 16:04:59 -0700
Subject: [PATCH 06/11] fix bug

---
 terraform-unity/evaluators/sns-sqs-lambda/data.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-unity/evaluators/sns-sqs-lambda/data.tf b/terraform-unity/evaluators/sns-sqs-lambda/data.tf
index 28995b4..587539f 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/data.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/data.tf
@@ -40,7 +40,7 @@ def lambda_handler(event, context):
             trace_id = generate_trace_id(start_time)
             segment_id = generate_segment_id()
         else:
-            d = { k:v for k,v in [i.split("=") for i in d.split(";")]}
+            d = { k:v for k,v in [i.split("=") for i in aws_trace_header.split(";")]}
             trace_id = d.get("Root", generate_trace_id(start_time))
             segment_id = d.get("Parent", generate_segment_id())
 

From 1e3cc316537753555af7f8b9cb255cbe80b7ecbf Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Tue, 20 Aug 2024 16:28:04 -0700
Subject: [PATCH 07/11] put annotation differently

---
 src/unity_initiator/cloud/lambda_handler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/unity_initiator/cloud/lambda_handler.py b/src/unity_initiator/cloud/lambda_handler.py
index 1bb75ba..1b32844 100644
--- a/src/unity_initiator/cloud/lambda_handler.py
+++ b/src/unity_initiator/cloud/lambda_handler.py
@@ -41,8 +41,8 @@ def lambda_handler_base(event, context):
                 f.write(router_cfg)
             ROUTER = Router(router_file)
             os.unlink(router_file)
-    with xray_recorder.capture("execute_actions") as subsegment:
-        subsegment.put_annotation("payload", event["payload"])
+    with xray_recorder.capture("execute_actions"):
+        xray_recorder.put_annotation("payload", event["payload"])
         return ROUTER.execute_actions(event["payload"])
 
 

From adb8061511699c3fc3d3373c473858c422d12f2d Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Wed, 21 Aug 2024 14:13:14 -0700
Subject: [PATCH 08/11] enable active tracing on SNS

---
 terraform-unity/evaluators/sns-sqs-lambda/main.tf | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/terraform-unity/evaluators/sns-sqs-lambda/main.tf b/terraform-unity/evaluators/sns-sqs-lambda/main.tf
index e406ec7..2718eaa 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/main.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/main.tf
@@ -127,7 +127,9 @@ resource "aws_sqs_queue" "evaluator_queue" {
 }
 
 resource "aws_sns_topic" "evaluator_topic" {
-  name = "${local.function_name}_topic"
+  name           = "${local.function_name}_topic"
+  tags           = local.tags
+  tracing_config = "Active"
 }
 
 resource "aws_sns_topic_policy" "evaluator_topic_policy" {

From 11d1d1032356f04962179778b8e0b0d908857158 Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Thu, 22 Aug 2024 08:59:43 -0700
Subject: [PATCH 09/11] use xray_recorder for tracing evaluator functions

---
 .../evaluators/sns-sqs-lambda/README.md       |  8 ++-
 .../sns-sqs-lambda/build_lambda_package.sh    | 23 +++++++
 .../evaluators/sns-sqs-lambda/data.tf         | 64 +------------------
 .../sns-sqs-lambda/lambda_handler.py          | 22 +++++++
 .../evaluators/sns-sqs-lambda/main.tf         | 43 ++++++++-----
 .../evaluators/sns-sqs-lambda/variables.tf    |  5 ++
 6 files changed, 86 insertions(+), 79 deletions(-)
 create mode 100755 terraform-unity/evaluators/sns-sqs-lambda/build_lambda_package.sh
 create mode 100644 terraform-unity/evaluators/sns-sqs-lambda/lambda_handler.py

diff --git a/terraform-unity/evaluators/sns-sqs-lambda/README.md b/terraform-unity/evaluators/sns-sqs-lambda/README.md
index a3c1339..4626485 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/README.md
+++ b/terraform-unity/evaluators/sns-sqs-lambda/README.md
@@ -15,8 +15,9 @@
 
 | Name | Version |
 |------|---------|
-| <a name="provider_archive"></a> [archive](#provider\_archive) | 2.4.2 |
 | <a name="provider_aws"></a> [aws](#provider\_aws) | 5.51.1 |
+| <a name="provider_local"></a> [local](#provider\_local) | 2.5.1 |
+| <a name="provider_null"></a> [null](#provider\_null) | 3.2.2 |
 
 ## Modules
 
@@ -34,6 +35,7 @@ No modules.
 | [aws_iam_role_policy_attachment.lambda_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_lambda_event_source_mapping.evaluator_queue_event_source_mapping](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource |
 | [aws_lambda_function.evaluator_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
+| [aws_s3_object.lambda_package](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_sns_topic.evaluator_topic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource |
 | [aws_sns_topic_policy.evaluator_topic_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_policy) | resource |
 | [aws_sns_topic_subscription.evaluator_subscription](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
@@ -41,14 +43,16 @@ No modules.
 | [aws_sqs_queue.evaluator_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
 | [aws_sqs_queue_policy.evaluator_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
 | [aws_ssm_parameter.evaluator_lambda_function_name](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ssm_parameter) | resource |
-| [archive_file.evaluator_lambda_artifact](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
+| [null_resource.build_lambda_package](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_iam_policy.mcp_operator_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source |
+| [local_file.version](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_code_bucket"></a> [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes |
 | <a name="input_evaluator_name"></a> [evaluator\_name](#input\_evaluator\_name) | The evaluator name | `string` | n/a | yes |
 | <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no |
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/build_lambda_package.sh b/terraform-unity/evaluators/sns-sqs-lambda/build_lambda_package.sh
new file mode 100755
index 0000000..d73b668
--- /dev/null
+++ b/terraform-unity/evaluators/sns-sqs-lambda/build_lambda_package.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+BASE_PATH=$(dirname "${BASH_SOURCE}")
+BASE_PATH=$(cd "${BASE_PATH}/../../.."; pwd)
+DIST_DIR=${BASE_PATH}/dist
+PKG_DIR=${DIST_DIR}/lambda_packages
+EVALUATOR_DIR=$(dirname "${BASH_SOURCE}")
+EVALUATOR_DIR=$(cd "${EVALUATOR_DIR}"; pwd)
+EVALUATOR_NAME=$1
+
+set -ex
+
+rm -rf $DIST_DIR
+pip install hatch
+hatch clean
+hatch build
+VERSION=$(hatch run python -c 'from importlib.metadata import version; print(version("unity_initiator"))')
+echo "{\"version\": \"$VERSION\"}" > ${DIST_DIR}/version.json
+mkdir -p $PKG_DIR
+pip install -t $PKG_DIR ${DIST_DIR}/unity_initiator-*.whl
+pip install -t $PKG_DIR aws_xray_sdk
+cp ${EVALUATOR_DIR}/lambda_handler.py $PKG_DIR/
+cd $PKG_DIR
+zip -rq ${DIST_DIR}/${EVALUATOR_NAME}-${VERSION}-lambda.zip .
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/data.tf b/terraform-unity/evaluators/sns-sqs-lambda/data.tf
index 587539f..4d6b523 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/data.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/data.tf
@@ -4,65 +4,7 @@ data "aws_iam_policy" "mcp_operator_policy" {
   name = "mcp-tenantOperator-AMI-APIG"
 }
 
-data "archive_file" "evaluator_lambda_artifact" {
-  type        = "zip"
-  output_path = "${path.root}/.archive_files/${var.evaluator_name}-evaluator_lambda.zip"
-
-  source {
-    filename = "lambda_function.py"
-    content  = <<CODE
-import time
-import os
-import binascii
-import json
-
-import boto3
-
-
-def generate_segment_id():
-    return binascii.b2a_hex(os.urandom(8)).decode()
-
-def generate_trace_id(start_time):
-    return "1-{}-{}".format(hex(int(start_time))[2:], binascii.hexlify(os.urandom(12)).decode('utf-8'))
-
-def lambda_handler(event, context):
-    print(f"event: {json.dumps(event, indent=2, sort_keys=True)}")
-    print(f"context: {context}")
-
-    client = boto3.client("xray")
-
-    for rec in event.get("Records", []):
-
-        # get trace info from AWSTraceHeader if set otherwise set them by default
-        start_time = time.time()
-        aws_trace_header = rec.get("attributes", {}).get("AWSTraceHeader", None)
-        if aws_trace_header is None:
-            trace_id = generate_trace_id(start_time)
-            segment_id = generate_segment_id()
-        else:
-            d = { k:v for k,v in [i.split("=") for i in aws_trace_header.split(";")]}
-            trace_id = d.get("Root", generate_trace_id(start_time))
-            segment_id = d.get("Parent", generate_segment_id())
-
-        # Implement your adaptation-specific evaluator code here and return
-        # True if it successfully evaluates. False otherwise.
-
-        # write trace
-        client.put_trace_segments(
-            TraceSegmentDocuments=[
-                json.dumps(
-                    {
-                        "name": context.function_name,
-                        "id": segment_id,
-                        "start_time": start_time,
-                        "trace_id": trace_id,
-                        "end_time": time.time(),
-                    }
-                )
-            ]
-        )
-
-    return { "success": True }
-CODE
-  }
+data "local_file" "version" {
+  filename   = "${path.module}/../../../dist/version.json"
+  depends_on = [null_resource.build_lambda_package]
 }
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/lambda_handler.py b/terraform-unity/evaluators/sns-sqs-lambda/lambda_handler.py
new file mode 100644
index 0000000..d6494bc
--- /dev/null
+++ b/terraform-unity/evaluators/sns-sqs-lambda/lambda_handler.py
@@ -0,0 +1,22 @@
+import json
+
+from aws_xray_sdk.core import patch_all, xray_recorder
+
+from unity_initiator.utils.logger import logger
+
+patch_all()
+
+
+def perform_evaluation(event, context):
+    logger.info("event: %s", json.dumps(event, indent=2))
+    logger.info("context: %s", context)
+
+    # Implement your adaptation-specific evaluator code here and return
+    # True if it successfully evaluates. False otherwise.
+
+    return True
+
+
+def lambda_handler(event, context):
+    with xray_recorder.capture(context.function_name):
+        return {"success": perform_evaluation(event, context)}
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/main.tf b/terraform-unity/evaluators/sns-sqs-lambda/main.tf
index 2718eaa..15d6cb7 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/main.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/main.tf
@@ -1,21 +1,32 @@
-# setup the actual lambda resource
-resource "aws_lambda_function" "evaluator_lambda" {
-  lifecycle {
-    # make sure tf doesn't overwrite deployed code once we start deploying
-    ignore_changes = [
-      filename,
-      source_code_hash,
-    ]
+resource "null_resource" "build_lambda_package" {
+  triggers = { always_run = timestamp() }
+  provisioner "local-exec" {
+    command = <<EOF
+      set -ex
+      docker run --rm -v ${path.module}/../../..:/var/task mlupin/docker-lambda:python3.9-build ./terraform-unity/evaluators/${basename(path.cwd)}/build_lambda_package.sh ${var.evaluator_name}
+    EOF
   }
+}
 
-  function_name    = local.function_name
-  role             = aws_iam_role.evaluator_lambda_iam_role.arn
-  handler          = "lambda_function.lambda_handler"
-  runtime          = "python3.11"
-  timeout          = 900
-  filename         = "${path.root}/.archive_files/${var.evaluator_name}-evaluator_lambda.zip"
-  source_code_hash = data.archive_file.evaluator_lambda_artifact.output_base64sha256
-  tags             = local.tags
+resource "aws_s3_object" "lambda_package" {
+  depends_on = [null_resource.build_lambda_package]
+  bucket     = var.code_bucket
+  key        = "${var.evaluator_name}-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  source     = "${path.module}/../../../dist/${var.evaluator_name}-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  etag       = filemd5("${path.module}/../../../dist/${var.evaluator_name}-${jsondecode(data.local_file.version.content).version}-lambda.zip")
+  tags       = local.tags
+}
+
+resource "aws_lambda_function" "evaluator_lambda" {
+  depends_on    = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.evaluator_lambda_log_group]
+  function_name = local.function_name
+  s3_bucket     = var.code_bucket
+  s3_key        = "${var.evaluator_name}-${jsondecode(data.local_file.version.content).version}-lambda.zip"
+  handler       = "lambda_handler.lambda_handler"
+  runtime       = "python3.11"
+  role          = aws_iam_role.evaluator_lambda_iam_role.arn
+  timeout       = 900
+  tags          = local.tags
 
   tracing_config {
     mode = "Active"
diff --git a/terraform-unity/evaluators/sns-sqs-lambda/variables.tf b/terraform-unity/evaluators/sns-sqs-lambda/variables.tf
index c95185b..796c66b 100644
--- a/terraform-unity/evaluators/sns-sqs-lambda/variables.tf
+++ b/terraform-unity/evaluators/sns-sqs-lambda/variables.tf
@@ -14,3 +14,8 @@ variable "venue" {
   type        = string
   default     = "dev"
 }
+
+variable "code_bucket" {
+  description = "The S3 bucket where lambda zip files will be stored and accessed"
+  type        = string
+}

From 2de1852729a46835d9c069d1e27111ef10b3863e Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Thu, 22 Aug 2024 09:20:54 -0700
Subject: [PATCH 10/11] Update README.md

---
 README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e5c6d52..12fdc69 100644
--- a/README.md
+++ b/README.md
@@ -307,7 +307,7 @@ In this demo we will deploy 2 evaluators:
 1. Note the implementation of the evaluator code. It currently doesn't do any real evaluation but simply returns that evaluation was successful:
 
    ```
-   cat data.tf
+   cat lambda_handler.py
    ```
 
 1. Initialize terraform:
@@ -321,6 +321,7 @@ In this demo we will deploy 2 evaluators:
    ```
    terraform apply \
      --var evaluator_name=${EVALUATOR_NAME} \
+     --var code_bucket=${CODE_BUCKET} \
      -auto-approve
    ```
 
@@ -349,7 +350,7 @@ In this demo we will deploy 2 evaluators:
 
 1. Note the implementation of the evaluator code. It currently doesn't do any real evaluation but simply returns that evaluation was successful:
    ```
-   cat data.tf
+   cat lambda_handler.py
    ```
 
 1. Initialize terraform:
@@ -361,6 +362,7 @@ In this demo we will deploy 2 evaluators:
    ```
    terraform apply \
      --var evaluator_name=${EVALUATOR_NAME} \
+     --var code_bucket=${CODE_BUCKET} \
      -auto-approve
    ```
 

From b6120cd2f874d3da06509580e43e6602e7846e69 Mon Sep 17 00:00:00 2001
From: Gerald Manipon <pymonger@gmail.com>
Date: Thu, 22 Aug 2024 13:51:24 -0700
Subject: [PATCH 11/11] capture trace using decorator

---
 src/unity_initiator/cloud/lambda_handler.py | 37 ++++++++++-----------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/unity_initiator/cloud/lambda_handler.py b/src/unity_initiator/cloud/lambda_handler.py
index 1b32844..e3c8b10 100644
--- a/src/unity_initiator/cloud/lambda_handler.py
+++ b/src/unity_initiator/cloud/lambda_handler.py
@@ -16,6 +16,7 @@
 ROUTER = None
 
 
+@xray_recorder.capture("lambda_handler_base")
 def lambda_handler_base(event, context):
     """Base lambda handler that instantiates a router, globally, and executes actions for a single payload."""
 
@@ -25,25 +26,23 @@ def lambda_handler_base(event, context):
     # or from a url in ROUTER_CFG_URL env variable.
     global ROUTER
     if ROUTER is None:
-        with xray_recorder.capture("read_router_config"):
-            router_cfg = os.environ.get("ROUTER_CFG", "").strip()
-            router_cfg_url = os.environ.get("ROUTER_CFG_URL", "").strip()
-            if router_cfg == "":
-                if router_cfg_url != "":
-                    with smart_open.open(router_cfg_url, "r") as f:
-                        router_cfg = f.read()
-                else:
-                    raise RuntimeError(
-                        "No router configuration specified via ROUTER_CFG or ROUTER_CFG_URL env variables."
-                    )
-            fd, router_file = mkstemp(prefix="router_", suffix=".yaml", text=True)
-            with os.fdopen(fd, "w") as f:
-                f.write(router_cfg)
-            ROUTER = Router(router_file)
-            os.unlink(router_file)
-    with xray_recorder.capture("execute_actions"):
-        xray_recorder.put_annotation("payload", event["payload"])
-        return ROUTER.execute_actions(event["payload"])
+        router_cfg = os.environ.get("ROUTER_CFG", "").strip()
+        router_cfg_url = os.environ.get("ROUTER_CFG_URL", "").strip()
+        if router_cfg == "":
+            if router_cfg_url != "":
+                with smart_open.open(router_cfg_url, "r") as f:
+                    router_cfg = f.read()
+            else:
+                raise RuntimeError(
+                    "No router configuration specified via ROUTER_CFG or ROUTER_CFG_URL env variables."
+                )
+        fd, router_file = mkstemp(prefix="router_", suffix=".yaml", text=True)
+        with os.fdopen(fd, "w") as f:
+            f.write(router_cfg)
+        ROUTER = Router(router_file)
+        os.unlink(router_file)
+    xray_recorder.put_annotation("payload", event["payload"])
+    return ROUTER.execute_actions(event["payload"])
 
 
 def lambda_handler_multiple_payloads(event, context):