Merge pull request #2 from unity-sds/413-submit-ogc

Add SRL Initiator for EDRGen and New OGC Action Type

src/unity_initiator/actions/submit_dag_by_id.py

@@ -34,7 +34,9 @@ def execute(self):
             },
             "note": "",
         }
-        response = httpx.post(url, auth=auth, headers=headers, json=body)
+        response = httpx.post(
+            url, auth=auth, headers=headers, json=body, verify=False
+        )  # nosec
         if response.status_code in (200, 201):
             success = True
             resp = response.json()

src/unity_initiator/actions/submit_ogc_process_execution.py

@@ -0,0 +1,50 @@
+import httpx
+
+from ..utils.logger import logger
+from .base import Action
+
+__all__ = ["SubmitOgcProcessExecution"]
+
+
+class SubmitOgcProcessExecution(Action):
+    def __init__(self, payload, payload_info, params):
+        super().__init__(payload, payload_info, params)
+        logger.info("instantiated %s", __class__.__name__)
+
+    def execute(self):
+        logger.debug("executing execute in %s", __class__.__name__)
+        url = f"{self._params['ogc_processes_base_api_endpoint']}/processes/{self._params['process_id']}/execution"
+        logger.info("url: %s", url)
+        headers = {"Content-Type": "application/json", "Accept": "application/json"}
+        # body = {
+        #     "inputs": self._params["execution_inputs"],
+        #     "outputs": self._params["execution_outputs"],
+        #     "subscriber": self._params["execution_subscriber"],
+        # }
+        body = {
+            "inputs": {
+                "payload": self._payload,
+                "payload_info": self._payload_info,
+                "on_success": self._params["on_success"],
+            },
+            "outputs": None,
+            "subscriber": None,
+        }
+        response = httpx.post(url, headers=headers, json=body, verify=False)  # nosec
+        if response.status_code in (200, 201):
+            success = True
+            resp = response.json()
+            logger.info(
+                "Successfully triggered the execution of the OGC Process %s: %s",
+                self._params["process_id"],
+                resp,
+            )
+        else:
+            success = False
+            resp = response.text
+            logger.info(
+                "Failed to trigger the execution of the OGC Process %s: %s",
+                self._params["process_id"],
+                resp,
+            )
+        return {"success": success, "response": resp}

src/unity_initiator/resources/routers_schema.yaml

@@ -25,7 +25,7 @@ initiator_config:
 # Configuration for matching payload (e.g. url) against a set of compiled regular expressions
 # and mapping any matches to a set of evaluators.
 regex_config:
-  regexes: list(compiled_regex(), required=True, min=1)
+  regexes: list(str, required=True, min=1)
   evaluators: list(include("evaluator_config"), required=True, min=1)
 
 # Configuration of actions that submit to evaluators.
@@ -36,7 +36,7 @@ evaluator_config:
 # Currently only 2 types of actions are supported:
 # 1. submit payload to an SNS topic
 # 2. submit payload to an airflow DAG
-action_config: any(include("submit_dag_by_id_action"), include("submit_to_sns_topic_action"))
+action_config: any(include("submit_dag_by_id_action"), include("submit_to_sns_topic_action"), include("submit_ogc_process_execution_action"))
 
 # Configuration for submitting a payload to an airflow DAG.
 submit_dag_by_id_action:
@@ -58,6 +58,17 @@ submit_to_sns_topic_action:
     topic_arn: str(required=False)
     on_success: include("on_success_actions", required=False)
 
+# Configuration for submitting a OGC process execution.
+submit_ogc_process_execution_action:
+  name: str(equals="submit_ogc_process_execution")
+  params:
+    process_id: str()
+    ogc_processes_base_api_endpoint: str(required=False)
+    execution_inputs: map(required=False)
+    execution_outputs: map(required=False)
+    execution_subscriber: map(required=False)
+    on_success: include("on_success_actions", required=False)
+
 # Configuration to pass onto the evaluator to use when evaluation is a success.
 on_success_actions:
-  actions: list(include("action_config"), required=True, min=1, max=1)
+  actions: list(include("action_config"), required=True, min=1, max=1)

src/unity_initiator/router.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import re
 
 from .evaluator import Evaluator
 from .utils.conf_utils import YamlConf, YamlConfEncoder
@@ -14,6 +15,15 @@ class Router:
     def __init__(self, config_file):
         self._config_file = config_file
         self._config = YamlConf(self._config_file)
+        self._compile_regexes()
+
+    def _compile_regexes(self):
+        """Compile all regex strings in the configuration."""
+        for url_cfg in (
+            self._config.get("initiator_config").get("payload_type").get("url", [])
+        ):
+            if "regexes" in url_cfg:
+                url_cfg["regexes"] = [re.compile(regex) for regex in url_cfg["regexes"]]
 
     def get_evaluators_by_url(self, url):
         found_match = False

terraform-unity/evaluators/sns_sqs_lambda/README.md → terraform-unity/evaluators/sns-sqs-lambda/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.8.2 |
 | <a name="requirement_archive"></a> [archive](#requirement\_archive) | >=2.4.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |

terraform-unity/triggers/scheduled_task/versions.tf → terraform-unity/evaluators/sns-sqs-lambda/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.6"
+  required_version = "~> 1.8.2"
 
   required_providers {
     archive = {

terraform-unity/initiator/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.8.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |
 | <a name="requirement_null"></a> [null](#requirement\_null) | >=3.2.2 |
@@ -26,7 +26,7 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [aws_cloudwatch_log_group.initiator_lambda_log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_cloudwatch_log_group.initiator_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_iam_policy.initiator_lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.initiator_lambda_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.lambda_base_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
@@ -35,7 +35,6 @@ No modules.
 | [aws_lambda_event_source_mapping.initiator_queue_event_source_mapping](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource |
 | [aws_lambda_function.initiator_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
 | [aws_s3_object.lambda_package](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
-| [aws_s3_object.router_config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource |
 | [aws_sns_topic.initiator_topic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource |
 | [aws_sns_topic_subscription.initiator_subscription](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
 | [aws_sqs_queue.initiator_dead_letter_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
@@ -51,10 +50,8 @@ No modules.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_code_bucket"></a> [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes |
-| <a name="input_config_bucket"></a> [config\_bucket](#input\_config\_bucket) | The S3 bucket where router configuration files will be stored and accessed | `string` | n/a | yes |
-| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes |
 | <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
-| <a name="input_router_config"></a> [router\_config](#input\_router\_config) | The local path to the router configuration file to use | `string` | n/a | yes |
+| <a name="input_router_config"></a> [router\_config](#input\_router\_config) | The S3 URL to the router configuration file | `string` | n/a | yes |
 | <a name="input_venue"></a> [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no |
 
 ## Outputs

terraform-unity/initiator/locals.tf

@@ -1,5 +1,5 @@
 locals {
-  function_name = "${var.project}-${var.venue}-${var.deployment_name}-inititator"
+  function_name = "${var.project}-${var.venue}-inititator"
   tags = {
     Venue       = "dev"
     ServiceArea = "cs"

terraform-unity/initiator/main.tf

@@ -3,7 +3,7 @@ resource "null_resource" "build_lambda_package" {
   provisioner "local-exec" {
     command = <<EOF
       set -ex
-      docker run --rm -v ${path.module}/../..:/var/task mlupin/docker-lambda:python3.9-build ./scripts/build_lambda_package.sh
+      docker run --rm -v $(realpath ${path.module}/../..):/var/task mlupin/docker-lambda:python3.9-build ./scripts/build_lambda_package.sh
     EOF
   }
 }
@@ -17,17 +17,9 @@ resource "aws_s3_object" "lambda_package" {
   tags       = local.tags
 }
 
-resource "aws_s3_object" "router_config" {
-  bucket = var.config_bucket
-  key    = basename(var.router_config)
-  source = var.router_config
-  etag   = filemd5(var.router_config)
-  tags   = local.tags
-}
-
 resource "aws_lambda_function" "initiator_lambda" {
-  depends_on    = [aws_s3_object.lambda_package, aws_s3_object.router_config, aws_cloudwatch_log_group.initiator_lambda_log_group]
-  function_name = local.function_name
+  depends_on    = [aws_s3_object.lambda_package, aws_cloudwatch_log_group.initiator_lambda]
+  function_name = "${var.project}-${var.venue}-inititator"
   s3_bucket     = var.code_bucket
   s3_key        = "unity_initiator-${jsondecode(data.local_file.version.content).version}-lambda.zip"
   handler       = "unity_initiator.cloud.lambda_handler.lambda_handler_initiator"
@@ -37,19 +29,20 @@ resource "aws_lambda_function" "initiator_lambda" {
 
   environment {
     variables = {
-      ROUTER_CFG_URL = "s3://${aws_s3_object.router_config.bucket}/${aws_s3_object.router_config.key}"
+      ROUTER_CFG_URL = var.router_config
     }
   }
   tags = local.tags
 }
 
-resource "aws_cloudwatch_log_group" "initiator_lambda_log_group" {
-  name              = "/aws/lambda/${local.function_name}"
+resource "aws_cloudwatch_log_group" "initiator_lambda" {
+  name              = "/aws/lambda/${var.project}-${var.venue}-inititator"
   retention_in_days = 14
+  tags              = local.tags
 }
 
 resource "aws_iam_role" "initiator_lambda_iam_role" {
-  name = "${var.project}-${var.venue}-${var.deployment_name}-initiator_lambda_iam_role"
+  name = "${var.project}-${var.venue}-initiator_lambda_iam_role"
 
   assume_role_policy = jsonencode({
     Version = "2012-10-17",
@@ -68,7 +61,7 @@ resource "aws_iam_role" "initiator_lambda_iam_role" {
 }
 
 resource "aws_iam_policy" "initiator_lambda_policy" {
-  name        = "${var.project}-${var.venue}-${var.deployment_name}-initiator_lambda_policy"
+  name        = "${var.project}-${var.venue}-initiator_lambda_policy"
   description = "A policy for the initiator lambda function to access S3 and SQS"
 
   policy = jsonencode({

terraform-unity/initiator/variables.tf

@@ -1,8 +1,3 @@
-variable "deployment_name" {
-  description = "The deployment name"
-  type        = string
-}
-
 variable "project" {
   description = "The unity project its installed into"
   type        = string
@@ -20,12 +15,7 @@ variable "code_bucket" {
   type        = string
 }
 
-variable "config_bucket" {
-  description = "The S3 bucket where router configuration files will be stored and accessed"
-  type        = string
-}
-
 variable "router_config" {
-  description = "The local path to the router configuration file to use"
+  description = "The S3 URL to the router configuration file"
   type        = string
-}
+}

terraform-unity/initiator/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.6"
+  required_version = "~> 1.8.2"
 
   required_providers {
     aws = {

terraform-unity/triggers/cmr_query/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.8.2 |
 | <a name="requirement_archive"></a> [archive](#requirement\_archive) | >=2.4.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |
@@ -52,7 +52,6 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_code_bucket"></a> [code\_bucket](#input\_code\_bucket) | The S3 bucket where lambda zip files will be stored and accessed | `string` | n/a | yes |
 | <a name="input_concept_id"></a> [concept\_id](#input\_concept\_id) | The concept ID for the data collection: https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#granule-search-by-parameters | `string` | n/a | yes |
-| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes |
 | <a name="input_initiator_topic_arn"></a> [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes |
 | <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
 | <a name="input_provider_id"></a> [provider\_id](#input\_provider\_id) | The short name for the data provider: https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#granule-search-by-parameters | `string` | n/a | yes |

terraform-unity/triggers/cmr_query/locals.tf

@@ -1,5 +1,5 @@
 locals {
-  function_name = "${var.project}-${var.venue}-${var.deployment_name}-cmr_query"
+  function_name = "${var.project}-${var.venue}-cmr_query"
   tags = {
     Venue       = "dev"
     ServiceArea = "cs"

terraform-unity/triggers/cmr_query/variables.tf

@@ -1,8 +1,3 @@
-variable "deployment_name" {
-  description = "The deployment name"
-  type        = string
-}
-
 variable "project" {
   description = "The unity project its installed into"
   type        = string

terraform-unity/triggers/cmr_query/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.6"
+  required_version = "~> 1.8.2"
 
   required_providers {
     archive = {

terraform-unity/triggers/s3_bucket_notification/README.md → terraform-unity/triggers/s3-bucket-notification/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.8.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |
 | <a name="requirement_null"></a> [null](#requirement\_null) | >=3.2.2 |

terraform-unity/triggers/scheduled_task/README.md → terraform-unity/triggers/scheduled-task/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.4.6 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | ~> 1.8.2 |
 | <a name="requirement_archive"></a> [archive](#requirement\_archive) | >=2.4.2 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >=5.50.0 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >=2.5.1 |
@@ -42,7 +42,6 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The deployment name | `string` | n/a | yes |
 | <a name="input_initiator_topic_arn"></a> [initiator\_topic\_arn](#input\_initiator\_topic\_arn) | The ARN of the initiator SNS topic to publish S3 events to | `string` | n/a | yes |
 | <a name="input_project"></a> [project](#input\_project) | The unity project its installed into | `string` | `"uod"` | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The unity venue its installed into | `string` | `"dev"` | no |

terraform-unity/triggers/scheduled_task/locals.tf → terraform-unity/triggers/scheduled-task/locals.tf

@@ -1,5 +1,5 @@
 locals {
-  function_name = "${var.project}-${var.venue}-${var.deployment_name}-scheduled_task"
+  function_name = "${var.project}-${var.venue}-scheduled_task"
   tags = {
     Venue       = "dev"
     ServiceArea = "cs"

terraform-unity/triggers/scheduled_task/main.tf → terraform-unity/triggers/scheduled-task/main.tf

@@ -50,7 +50,7 @@ resource "aws_cloudwatch_log_group" "scheduled_task_lambda_log_group" {
 }
 
 resource "aws_iam_role" "scheduler" {
-  name = "${var.project}-${var.venue}-${var.deployment_name}-cron-scheduler-role"
+  name = "${var.project}-${var.venue}-cron-scheduler-role"
   assume_role_policy = jsonencode({
     Version = "2012-10-17"
     Statement = [
@@ -68,7 +68,7 @@ resource "aws_iam_role" "scheduler" {
 }
 
 resource "aws_iam_policy" "scheduler" {
-  name = "${var.project}-${var.venue}-${var.deployment_name}-cron-scheduler-policy"
+  name = "${var.project}-${var.venue}-cron-scheduler-policy"
   policy = jsonencode({
     Version = "2012-10-17"
     Statement = [
@@ -90,7 +90,7 @@ resource "aws_iam_role_policy_attachment" "scheduler" {
 }
 
 resource "aws_scheduler_schedule" "run_scheduled_task" {
-  name                = "${var.project}-${var.venue}-${var.deployment_name}-run_scheduled_task"
+  name                = "${var.project}-${var.venue}-run_scheduled_task"
   schedule_expression = "rate(1 minute)"
   flexible_time_window {
     mode = "OFF"

terraform-unity/triggers/scheduled_task/variables.tf → terraform-unity/triggers/scheduled-task/variables.tf

@@ -1,8 +1,3 @@
-variable "deployment_name" {
-  description = "The deployment name"
-  type        = string
-}
-
 variable "project" {
   description = "The unity project its installed into"
   type        = string

terraform-unity/evaluators/sns_sqs_lambda/versions.tf → terraform-unity/triggers/scheduled-task/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.6"
+  required_version = "~> 1.8.2"
 
   required_providers {
     archive = {