update production to ab28140 - ready to turn on airflow (#276)

* update production to ab28140 - ready to turn on airflow

* add ENVIRONMENT to dags

* add v1 in order to be able to force chagne to docker compose

terraform/modules/services/airflow/dags/dayafter-dag.py

@@ -2,6 +2,7 @@
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
 from airflow.decorators import dag
+import os
 
 from airflow.operators.latest_only import LatestOnlyOperator
 
@@ -17,7 +18,8 @@
     "max_active_tasks": 10,
 }
 
-cluster = "Nowcasting-development"
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/dags/forecast-gsp-dag.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
@@ -16,7 +17,8 @@
     'max_active_tasks':10,
 }
 
-cluster = 'Nowcasting-development'
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/dags/forecast-national-dag.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
@@ -16,7 +17,8 @@
     'max_active_tasks':10,
 }
 
-cluster = 'Nowcasting-development'
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/dags/gsp-dag.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
@@ -16,7 +17,8 @@
     'max_active_tasks':10,
 }
 
-cluster = 'Nowcasting-development'
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/dags/nwp-dag.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
@@ -16,7 +17,8 @@
     'max_active_tasks':10,
 }
 
-cluster = 'Nowcasting-development'
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/dags/pv-dag.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
@@ -16,7 +17,8 @@
     'max_active_tasks':10,
 }
 
-cluster = 'Nowcasting-development'
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/dags/satellite-dag.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime, timedelta
 from airflow import DAG
 from airflow.providers.amazon.aws.operators.ecs import EcsRunTaskOperator
@@ -15,7 +16,8 @@
     "max_active_tasks": 10,
 }
 
-cluster = "Nowcasting-development"
+env = os.getenv("ENVIRONMENT","development")
+cluster = f"Nowcasting-{env}"
 
 # Tasks can still be defined in terraform, or defined here
 

terraform/modules/services/airflow/docker-compose.yml

@@ -41,6 +41,7 @@ services:
       AIRFLOW__LOGGING__BASE_LOG_FOLDER: "/airflow/logs"
       AIRFLOW__LOGGING__LOGGING_LEVEL: $LOGLEVEL
       AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
+      ENVIRONMENT: $ENVIRONMENT
     user: "${AIRFLOW_UID:-50000}:0"
     volumes:
       - data:/airflow
@@ -69,6 +70,7 @@ services:
       AIRFLOW__CORE__DAGS_FOLDER: "/airflow/dags"
       AIRFLOW__LOGGING__BASE_LOG_FOLDER: "/airflow/logs"
       AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
+      ENVIRONMENT: $ENVIRONMENT
     user: "${AIRFLOW_UID:-50000}:0"
     volumes:
       - data:/airflow

terraform/modules/services/airflow/s3.tf

@@ -6,7 +6,7 @@ resource "aws_s3_bucket" "airflow-s3" {
 
 resource "aws_s3_object" "eb_object" {
   bucket = aws_s3_bucket.airflow-s3.id
-  key = "beanstalk/docker-compose-${var.docker-compose-version}.yml"
+  key = "beanstalk/docker-compose-${var.docker-compose-version}-v1.yml"
   source = "${path.module}/docker-compose.yml"
 }
 

terraform/nowcasting/production/main.tf

@@ -9,7 +9,7 @@ locals {
 
 
 module "networking" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/networking?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/networking?ref=ab28140"
   region               = var.region
   environment          = var.environment
   vpc_cidr             = var.vpc_cidr
@@ -27,22 +27,22 @@ module "ec2-bastion" {
 }
 
 module "s3" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-trio?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-trio?ref=ab28140"
 
   region      = var.region
   environment = var.environment
 
 }
 
 module "ecs" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/ecs?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/ecs?ref=ab28140"
   region      = var.region
   environment = var.environment
   domain = local.domain
 }
 
 module "forecasting_models_bucket" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-private?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/s3-private?ref=ab28140"
 
   region              = var.region
   environment         = var.environment
@@ -52,7 +52,7 @@ module "forecasting_models_bucket" {
 }
 
 module "api" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/api?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/api?ref=ab28140"
 
   region                              = var.region
   environment                         = var.environment
@@ -72,7 +72,7 @@ module "api" {
 
 
 module "database" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/database-pair?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/database-pair?ref=ab28140"
 
   region          = var.region
   environment     = var.environment
@@ -81,7 +81,7 @@ module "database" {
 }
 
 module "nwp" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=ab28140"
 
   region                  = var.region
   environment             = var.environment
@@ -100,7 +100,7 @@ module "nwp" {
 }
 
 module "nwp-national" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/nwp?ref=ab28140"
 
   region                  = var.region
   environment             = var.environment
@@ -119,7 +119,7 @@ module "nwp-national" {
 }
 
 module "sat" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/sat?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/sat?ref=ab28140"
 
   region                  = var.region
   environment             = var.environment
@@ -134,7 +134,7 @@ module "sat" {
 
 
 module "pv" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/pv?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/pv?ref=ab28140"
 
   region                  = var.region
   environment             = var.environment
@@ -149,7 +149,7 @@ module "pv" {
 }
 
 module "gsp" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/gsp?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/gsp?ref=ab28140"
 
   region                  = var.region
   environment             = var.environment
@@ -161,7 +161,7 @@ module "gsp" {
 }
 
 module "metrics" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/metrics?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/metrics?ref=ab28140"
 
   region                  = var.region
   environment             = var.environment
@@ -174,7 +174,7 @@ module "metrics" {
 
 
 module "forecast" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast?ref=ab28140"
 
   region                        = var.region
   environment                   = var.environment
@@ -195,7 +195,7 @@ module "forecast" {
 
 
 module "national_forecast" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=eac7811"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=ab28140"
 
   region      = var.region
   environment = var.environment
@@ -227,7 +227,7 @@ module "national_forecast" {
 }
 
 module "analysis_dashboard" {
-    source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/internal_ui?ref=eac7811"
+    source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/internal_ui?ref=ab28140"
 
     region      = var.region
     environment = var.environment
@@ -250,7 +250,7 @@ module "analysis_dashboard" {
 
 
 module "forecast_pvnet" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=0531df0"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=ab28140"
 
   region      = var.region
   environment = var.environment