Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hotfix/eis fire vector support #244

Closed
wants to merge 7 commits into from
Closed

Hotfix/eis fire vector support #244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
03c35d9
Merge pull request #198 from NASA-IMPACT/dev
ranchodeluxe Aug 12, 2024
574a173
Merge pull request #218 from NASA-IMPACT/dev
ranchodeluxe Aug 13, 2024
7e3e134
Merge pull request #235 from NASA-IMPACT/dev
smohiudd Oct 17, 2024
2810e67
the generic vector pipeline should not block eis vector processing
anayeaye Oct 18, 2024
98702d6
fix tag filter
smohiudd Oct 18, 2024
cd42b70
update vector subnet ref
smohiudd Oct 18, 2024
bd1d605
fix vector subnet ids
smohiudd Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions dags/veda_data_pipeline/groups/discover_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,13 @@ def vector_raster_choice(ti):
payload = ti.dag_run.conf
dynamic_group_id = ti.task_id.split(".")[0]

if payload.get("vector"):
# New generic pipeline payloads should be identified as generic
if payload.get("vector_generic"):
return f"{dynamic_group_id}.parallel_run_process_generic_vectors"
if payload.get("vector_eis"):
# Existing EIS fire vector paloyds should be processed as usual
if payload.get("vector"):
return f"{dynamic_group_id}.parallel_run_process_vectors"

return f"{dynamic_group_id}.parallel_run_process_rasters"

@task_group
Expand Down
8 changes: 4 additions & 4 deletions infrastructure/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,14 @@ module "custom_policy" {
vector_secret_name = var.vector_secret_name
}

data "aws_subnets" "private" {
data "aws_subnets" "vector_aws_subnets" {
filter {
name = "vpc-id"
values = [var.vector_vpc == null ? "" : var.vector_vpc]
}

tags = {
"Scope" = "private"
Scope = "private"
}
}

Expand Down Expand Up @@ -109,8 +109,8 @@ resource "local_file" "mwaa_variables" {
stac_ingestor_api_url = var.stac_ingestor_api_url
stac_url = var.stac_url
vector_secret_name = var.vector_secret_name
vector_subnet_1 = length(data.aws_subnets.subnet_ids.ids) > 0 ? data.aws_subnets.subnet_ids.ids[0] : ""
vector_subnet_2 = length(data.aws_subnets.subnet_ids.ids) > 0 ? data.aws_subnets.subnet_ids.ids[1] : ""
vector_subnet_1 = length(data.aws_subnets.vector_aws_subnets.ids) > 0 ? data.aws_subnets.vector_aws_subnets.ids[0] : ""
vector_subnet_2 = length(data.aws_subnets.vector_aws_subnets.ids) > 0 ? data.aws_subnets.vector_aws_subnets.ids[1] : ""
vector_security_group = length(aws_security_group.vector_sg) > 0 ? aws_security_group.vector_sg[0].id : ""
vector_vpc = var.vector_vpc
})
Expand Down