diff --git a/terraform/modules/services/airflow/dags/dayafter-dag.py b/terraform/modules/services/airflow/dags/dayafter-dag.py index b34b5048..a0f4aec1 100644 --- a/terraform/modules/services/airflow/dags/dayafter-dag.py +++ b/terraform/modules/services/airflow/dags/dayafter-dag.py @@ -2,6 +2,7 @@ from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator from airflow.decorators import dag +import os from airflow.operators.latest_only import LatestOnlyOperator @@ -17,7 +18,8 @@ "max_active_tasks": 10, } -cluster = "Nowcasting-development" +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/dags/forecast-gsp-dag.py b/terraform/modules/services/airflow/dags/forecast-gsp-dag.py index 2a523240..0b2e8d01 100644 --- a/terraform/modules/services/airflow/dags/forecast-gsp-dag.py +++ b/terraform/modules/services/airflow/dags/forecast-gsp-dag.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator @@ -16,7 +17,8 @@ 'max_active_tasks':10, } -cluster = 'Nowcasting-development' +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/dags/forecast-national-dag.py b/terraform/modules/services/airflow/dags/forecast-national-dag.py index c7416b99..f7f827bd 100644 --- a/terraform/modules/services/airflow/dags/forecast-national-dag.py +++ b/terraform/modules/services/airflow/dags/forecast-national-dag.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator @@ -16,7 +17,8 @@ 'max_active_tasks':10, } -cluster = 'Nowcasting-development' +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/dags/gsp-dag.py b/terraform/modules/services/airflow/dags/gsp-dag.py index 67394bb6..43d5ac7f 100644 --- a/terraform/modules/services/airflow/dags/gsp-dag.py +++ b/terraform/modules/services/airflow/dags/gsp-dag.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator @@ -16,7 +17,8 @@ 'max_active_tasks':10, } -cluster = 'Nowcasting-development' +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/dags/nwp-dag.py b/terraform/modules/services/airflow/dags/nwp-dag.py index bde1e5e3..c84ec23e 100644 --- a/terraform/modules/services/airflow/dags/nwp-dag.py +++ b/terraform/modules/services/airflow/dags/nwp-dag.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator @@ -16,7 +17,8 @@ 'max_active_tasks':10, } -cluster = 'Nowcasting-development' +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/dags/pv-dag.py b/terraform/modules/services/airflow/dags/pv-dag.py index 6b460717..e0e70cef 100644 --- a/terraform/modules/services/airflow/dags/pv-dag.py +++ b/terraform/modules/services/airflow/dags/pv-dag.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator @@ -16,7 +17,8 @@ 'max_active_tasks':10, } -cluster = 'Nowcasting-development' +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/dags/satellite-dag.py b/terraform/modules/services/airflow/dags/satellite-dag.py index 0436fa59..1f02c8b0 100644 --- a/terraform/modules/services/airflow/dags/satellite-dag.py +++ b/terraform/modules/services/airflow/dags/satellite-dag.py @@ -1,3 +1,4 @@ +import os from datetime import datetime, timedelta from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator @@ -15,7 +16,8 @@ "max_active_tasks": 10, } -cluster = "Nowcasting-development" +env = os.getenv("ENVIRONMENT","development") +cluster = f"Nowcasting-{env}" # Tasks can still be defined in terraform, or defined here diff --git a/terraform/modules/services/airflow/docker-compose.yml b/terraform/modules/services/airflow/docker-compose.yml index 9db5ec59..c83ab0bc 100644 --- a/terraform/modules/services/airflow/docker-compose.yml +++ b/terraform/modules/services/airflow/docker-compose.yml @@ -41,6 +41,7 @@ services: AIRFLOW__LOGGING__BASE_LOG_FOLDER: "/airflow/logs" AIRFLOW__LOGGING__LOGGING_LEVEL: $LOGLEVEL AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION + ENVIRONMENT: $ENVIRONMENT user: "${AIRFLOW_UID:-50000}:0" volumes: - data:/airflow @@ -69,6 +70,7 @@ services: AIRFLOW__CORE__DAGS_FOLDER: "/airflow/dags" AIRFLOW__LOGGING__BASE_LOG_FOLDER: "/airflow/logs" AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION + ENVIRONMENT: $ENVIRONMENT user: "${AIRFLOW_UID:-50000}:0" volumes: - data:/airflow diff --git a/terraform/modules/services/airflow/s3.tf b/terraform/modules/services/airflow/s3.tf index 5dabaad2..5fd543f6 100644 --- a/terraform/modules/services/airflow/s3.tf +++ b/terraform/modules/services/airflow/s3.tf @@ -6,7 +6,7 @@ resource "aws_s3_bucket" "airflow-s3" { resource "aws_s3_object" "eb_object" { bucket = aws_s3_bucket.airflow-s3.id - key = "beanstalk/docker-compose-${var.docker-compose-version}.yml" + key = "beanstalk/docker-compose-${var.docker-compose-version}-v1.yml" source = "${path.module}/docker-compose.yml" } diff --git a/terraform/nowcasting/production/main.tf b/terraform/nowcasting/production/main.tf index d3d8dfe5..2300b885 100644 --- a/terraform/nowcasting/production/main.tf +++ b/terraform/nowcasting/production/main.tf @@ -9,7 +9,7 @@ locals { module "networking" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/networking?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/networking?ref=ab28140" region = var.region environment = var.environment vpc_cidr = var.vpc_cidr @@ -27,7 +27,7 @@ module "ec2-bastion" { } module "s3" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-trio?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-trio?ref=ab28140" region = var.region environment = var.environment @@ -35,14 +35,14 @@ module "s3" { } module "ecs" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/ecs?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/ecs?ref=ab28140" region = var.region environment = var.environment domain = local.domain } module "forecasting_models_bucket" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-private?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-private?ref=ab28140" region = var.region environment = var.environment @@ -52,7 +52,7 @@ module "forecasting_models_bucket" { } module "api" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/api?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/api?ref=ab28140" region = var.region environment = var.environment @@ -72,7 +72,7 @@ module "api" { module "database" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/database-pair?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/database-pair?ref=ab28140" region = var.region environment = var.environment @@ -81,7 +81,7 @@ module "database" { } module "nwp" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=ab28140" region = var.region environment = var.environment @@ -100,7 +100,7 @@ module "nwp" { } module "nwp-national" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=ab28140" region = var.region environment = var.environment @@ -119,7 +119,7 @@ module "nwp-national" { } module "sat" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/sat?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/sat?ref=ab28140" region = var.region environment = var.environment @@ -134,7 +134,7 @@ module "sat" { module "pv" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/pv?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/pv?ref=ab28140" region = var.region environment = var.environment @@ -149,7 +149,7 @@ module "pv" { } module "gsp" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/gsp?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/gsp?ref=ab28140" region = var.region environment = var.environment @@ -161,7 +161,7 @@ module "gsp" { } module "metrics" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/metrics?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/metrics?ref=ab28140" region = var.region environment = var.environment @@ -174,7 +174,7 @@ module "metrics" { module "forecast" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast?ref=ab28140" region = var.region environment = var.environment @@ -195,7 +195,7 @@ module "forecast" { module "national_forecast" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=ab28140" region = var.region environment = var.environment @@ -227,7 +227,7 @@ module "national_forecast" { } module "analysis_dashboard" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/internal_ui?ref=eac7811" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/internal_ui?ref=ab28140" region = var.region environment = var.environment @@ -250,7 +250,7 @@ module "analysis_dashboard" { module "forecast_pvnet" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=0531df0" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=ab28140" region = var.region environment = var.environment