diff --git a/dags/veda_data_pipeline/groups/discover_group.py b/dags/veda_data_pipeline/groups/discover_group.py index 798f5f10..9ac81f60 100644 --- a/dags/veda_data_pipeline/groups/discover_group.py +++ b/dags/veda_data_pipeline/groups/discover_group.py @@ -70,10 +70,13 @@ def vector_raster_choice(ti): payload = ti.dag_run.conf dynamic_group_id = ti.task_id.split(".")[0] - if payload.get("vector"): + # New generic pipeline payloads should be identified as generic + if payload.get("vector_generic"): return f"{dynamic_group_id}.parallel_run_process_generic_vectors" - if payload.get("vector_eis"): + # Existing EIS fire vector paloyds should be processed as usual + if payload.get("vector"): return f"{dynamic_group_id}.parallel_run_process_vectors" + return f"{dynamic_group_id}.parallel_run_process_rasters" @task_group diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 0f22eb64..83a4ae76 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -54,14 +54,14 @@ module "custom_policy" { vector_secret_name = var.vector_secret_name } -data "aws_subnets" "private" { +data "aws_subnets" "vector_aws_subnets" { filter { name = "vpc-id" values = [var.vector_vpc == null ? "" : var.vector_vpc] } tags = { - "Scope" = "private" + Scope = "private" } } @@ -109,8 +109,8 @@ resource "local_file" "mwaa_variables" { stac_ingestor_api_url = var.stac_ingestor_api_url stac_url = var.stac_url vector_secret_name = var.vector_secret_name - vector_subnet_1 = length(data.aws_subnets.subnet_ids.ids) > 0 ? data.aws_subnets.subnet_ids.ids[0] : "" - vector_subnet_2 = length(data.aws_subnets.subnet_ids.ids) > 0 ? data.aws_subnets.subnet_ids.ids[1] : "" + vector_subnet_1 = length(data.aws_subnets.vector_aws_subnets.ids) > 0 ? data.aws_subnets.vector_aws_subnets.ids[0] : "" + vector_subnet_2 = length(data.aws_subnets.vector_aws_subnets.ids) > 0 ? data.aws_subnets.vector_aws_subnets.ids[1] : "" vector_security_group = length(aws_security_group.vector_sg) > 0 ? aws_security_group.vector_sg[0].id : "" vector_vpc = var.vector_vpc })