terraform: add data_import_env_vars to aws env

data/README.md

@@ -8,6 +8,10 @@ Seed data for the API can be loaded using the included Makefile. You can execute
 - To run the seed data importer natively, run `make seed-data`
 - To run the seed data importer in a docker container, use the convenient `./data.sh seed-data` script.
 
-When natively executing the import, you need to manually set up dependencies, like `python` packages (see `requirements.txt`) as well as other system dependencies (see `Dockerfile`). You also need to manually set several env vars to allow the application to connect to the Postgres database that will host the data.
+When natively executing the import, you need to manually set up dependencies, like `python` packages (
+see `requirements.txt`) as well as other system dependencies (see `Dockerfile`). You also need to manually set several
+env vars to allow the application to connect to the Postgres database that will host the data.
 
-If running using a docker container, application dependencies will be automatically installed for you (you do need to have docker and docker-compose properly set up) and env vars will be loaded from the `.env` file at the root of the project - see the `env.default` file for an example/skeleton of the `.env` file you'll need to have.
+If running using a docker container, application dependencies will be automatically installed for you (you do need to
+have docker and docker-compose properly set up) and env vars will be loaded from the `.env` file at the root of the
+project - see the `env.default` file for an example/skeleton of the `.env` file you'll need to have.

infrastructure/kubernetes/main.tf

@@ -9,7 +9,7 @@ terraform {
 
 data "terraform_remote_state" "core" {
   backend = "s3"
-  config  = {
+  config = {
     bucket = var.tf_state_bucket
     region = var.aws_region
     key    = "core.tfstate"
@@ -56,14 +56,14 @@ resource "github_actions_secret" "mapbox_api_token_secret" {
 module "aws_environment" {
   for_each = merge(var.aws_environments, {
     staging = merge({
-      load_fresh_data       = false
+      load_fresh_data = false
       data_import_arguments = ["seed-data"]
-      image_tag             = "staging"
+      image_tag       = "staging"
     }, lookup(var.aws_environments, "staging", {})),
     production = merge({
-      load_fresh_data       = false
+      load_fresh_data = false
       data_import_arguments = ["seed-data"]
-      image_tag             = "main"
+      image_tag       = "main"
     }, lookup(var.aws_environments, "production", {})),
   })
   source = "./modules/aws/env"
@@ -76,20 +76,21 @@ module "aws_environment" {
   allowed_account_id                 = var.allowed_account_id
   gmaps_api_key                      = var.gmaps_api_key
   sendgrid_api_key                   = var.sendgrid_api_key
-  eudr_credentials                   = jsonencode(var.eudr_credentials)
-  load_fresh_data                    = lookup(each.value, "load_fresh_data", false)
-  data_import_arguments              = lookup(each.value, "data_import_arguments", ["seed-data"])
-  image_tag                          = lookup(each.value, "image_tag", each.key)
-  repo_branch                        = lookup(each.value, "image_tag", each.key)
+  eudr_credentials = jsonencode(var.eudr_credentials)
+  load_fresh_data = lookup(each.value, "load_fresh_data", false)
+  data_import_arguments = lookup(each.value, "data_import_arguments", ["seed-data"])
+  image_tag = lookup(each.value, "image_tag", each.key)
+  repo_branch = lookup(each.value, "image_tag", each.key)
   private_subnet_ids                 = data.terraform_remote_state.core.outputs.private_subnet_ids
   repo_name                          = var.repo_name
   domain                             = var.domain
   api_container_registry_url         = data.terraform_remote_state.core.outputs.aws_api_container_registry_url
   client_container_registry_url      = data.terraform_remote_state.core.outputs.aws_client_container_registry_url
   tiler_container_registry_url       = data.terraform_remote_state.core.outputs.aws_tiler_container_registry_url
   data_import_container_registry_url = data.terraform_remote_state.core.outputs.aws_data_import_container_registry_url
-  api_env_vars                       = lookup(each.value, "api_env_vars", [])
-  api_secrets                        = lookup(each.value, "api_secrets", [])
+  data_import_env_vars = lookup(each.value, "data_import_env_vars", [])
+  api_env_vars = lookup(each.value, "api_env_vars", [])
+  api_secrets = lookup(each.value, "api_secrets", [])
   science_bucket_name                = data.terraform_remote_state.core.outputs.science_bucket_name
 
   providers = {
@@ -108,19 +109,19 @@ module "gcp_environment" {
   tf_state_bucket                    = var.tf_state_bucket
   allowed_account_id                 = var.allowed_account_id
   gmaps_api_key                      = var.gmaps_api_key
-  load_fresh_data                    = lookup(each.value, "load_fresh_data", false)
-  data_import_arguments              = lookup(each.value, "data_import_arguments", ["seed-data"])
-  image_tag                          = lookup(each.value, "image_tag", each.key)
-  repo_branch                        = lookup(each.value, "repo_branch", each.key)
+  load_fresh_data = lookup(each.value, "load_fresh_data", false)
+  data_import_arguments = lookup(each.value, "data_import_arguments", ["seed-data"])
+  image_tag = lookup(each.value, "image_tag", each.key)
+  repo_branch = lookup(each.value, "repo_branch", each.key)
   private_subnet_ids                 = data.terraform_remote_state.core.outputs.private_subnet_ids
   repo_name                          = var.repo_name
   domain                             = var.domain
   api_container_registry_url         = data.terraform_remote_state.core.outputs.gcp_api_container_registry_url
   client_container_registry_url      = data.terraform_remote_state.core.outputs.gcp_client_container_registry_url
   tiler_container_registry_url       = data.terraform_remote_state.core.outputs.gcp_tiler_container_registry_url
   data_import_container_registry_url = data.terraform_remote_state.core.outputs.gcp_data_import_container_registry_url
-  api_env_vars                       = lookup(each.value, "api_env_vars", [])
-  api_secrets                        = lookup(each.value, "api_secrets", [])
+  api_env_vars = lookup(each.value, "api_env_vars", [])
+  api_secrets = lookup(each.value, "api_secrets", [])
   science_bucket_name                = data.terraform_remote_state.core.outputs.science_bucket_name
   gcp_project                        = var.gcp_project_id
   gcp_region                         = var.gcp_region

infrastructure/kubernetes/modules/aws/env/main.tf

@@ -43,7 +43,7 @@ locals {
     for env in var.api_env_vars : env.name => env.value
   }
   api_env_vars_map = merge(local.default_api_env_vars_map, local.overlapping_api_env_vars_map)
-  api_env_vars     = [
+  api_env_vars = [
     for name, value in local.api_env_vars_map : {
       name  = name
       value = value
@@ -168,7 +168,7 @@ module "k8s_tiler" {
       value = "${module.k8s_api.api_service_name}.${var.environment}.svc.cluster.local"
     },
     {
-      name  = "API_PORT"
+      name = "API_PORT"
       // TODO: get port from api k8s service
       value = 3000
     },
@@ -224,7 +224,7 @@ module "k8s_data_import" {
   load_data = var.load_fresh_data
   arguments = var.data_import_arguments
 
-  env_vars = [
+  env_vars = concat(var.data_import_env_vars, [
     {
       name  = "API_POSTGRES_PORT"
       value = "5432"
@@ -237,7 +237,7 @@ module "k8s_data_import" {
       name  = "S3_COG_PATH"
       value = "processed/cogs"
     },
-  ]
+  ])
 
   secrets = [
     {
@@ -305,8 +305,8 @@ module "data-import-group" {
   max_size           = 2
   desired_size       = 1
   namespace          = var.environment
-  subnet_ids         = [var.private_subnet_ids[0]]
-  labels             = {
+  subnet_ids = [var.private_subnet_ids[0]]
+  labels = {
     type : "data-import-${var.environment}"
   }
 }

infrastructure/kubernetes/modules/aws/env/variables.tf

@@ -36,7 +36,7 @@ variable "domain" {
 }
 
 variable "private_subnet_ids" {
-  type        = list(string)
+  type = list(string)
   description = "IDs of the subnets used in the EKS cluster"
 }
 
@@ -80,8 +80,8 @@ variable "load_fresh_data" {
 }
 
 variable "data_import_arguments" {
-  type        = list(string)
-  default     = ["seed-data"]
+  type = list(string)
+  default = ["seed-data"]
   description = "Arguments to pass to the initial data import process"
 }
 
@@ -116,7 +116,7 @@ variable "api_env_vars" {
     value = string
   }))
   description = "Key-value pairs of env vars to make available to the api container"
-  default     = []
+  default = []
 }
 
 
@@ -127,7 +127,7 @@ variable "api_secrets" {
     secret_key  = string
   }))
   description = "List of secrets to make available to the api container"
-  default     = []
+  default = []
 }
 
 
@@ -137,5 +137,14 @@ variable "tiler_env_vars" {
     value = string
   }))
   description = "Key-value pairs of env vars to make available to the tiler container"
-  default     = []
+  default = []
+}
+
+variable data_import_env_vars {
+  type = list(object({
+    name  = string
+    value = any
+  }))
+  description = "Key-value pairs of env vars to make available to the data import container"
+  default = []
 }