Remove store AWS access credentials on TF; Remove AWS access env var …

…validation on data import process
Vizzuality · Nov 21, 2022 · d1ff85f · d1ff85f
1 parent 0943244
commit d1ff85f
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 59 deletions.
diff --git a/data/h3_data_importer/Makefile b/data/h3_data_importer/Makefile
@@ -6,6 +6,7 @@
 #########################
 PARALLELIZATION_FACTOR=10
 
+AWS_S3_BUCKET_URL=s3://landgriffon-raw-data
 WORKDIR_MAPSPAM=data/mapspam
 WORKDIR_GLW3=data/glw3
 WORKDIR_WATER_FOOTPRINT=data/waterFootprint
@@ -18,16 +19,16 @@ WORKDIR_HDI=data/contextual/hdi
 WORKDIR_GHG=data/forest_ghg
 WORKDIR_WOODPULP=data/woodpulp
 
-export AWS_ACCESS_KEY_ID = $(DATA_S3_ACCESS_KEY)
-export AWS_SECRET_ACCESS_KEY = $(DATA_S3_SECRET_KEY)
-
 all:
-ifeq ($(DATA_S3_ACCESS_KEY),)
-	$(error AWS S3 credentials are not defined in env vars. Import will not run)
-else
+	@aws s3 ls $(AWS_S3_BUCKET_URL) 2>&1 > /dev/null; \
+	if [ $$? -ne 0 ]; \
+	then \
+		echo "Error: Cannot access the expected AWS S3 bucket."; \
+		false; \
+	fi
+	@echo "AWS S3 bucket access confirmed, proceeding with import..."
 	make clean
 	make -j 2 crop indicators
-endif
 
 crop:
 	make -j 2 convert-mapspam-crop-production convert-mapspam-crop-harvest convert-glw3-livestock convert-woodpulp
@@ -436,7 +437,7 @@ convert-aboveGroundBiomass: preprocess-aboveGroundBiomass
 download-satDeforestation:
 	mkdir -p $(WORKDIR_SATDEFORESTATION)
 
-	aws s3 sync s3://landgriffon-raw-data/processed/satelligence $(WORKDIR_SATDEFORESTATION)
+	aws s3 sync $(AWS_S3_BUCKET_URL)/processed/satelligence $(WORKDIR_SATDEFORESTATION)
 
 	mkdir -p $(WORKDIR_SATDEFORESTATION)/mask
 	mkdir -p $(WORKDIR_SATDEFORESTATION)/risk
@@ -456,7 +457,7 @@ download-woodpulp:
 	mkdir -p $(WORKDIR_WOODPULP)/ha
 	mkdir -p $(WORKDIR_WOODPULP)/prod
 
-	aws s3 sync s3://landgriffon-raw-data/processed/woodpulp/ $(WORKDIR_WOODPULP)
+	aws s3 sync $(AWS_S3_BUCKET_URL)/processed/woodpulp/ $(WORKDIR_WOODPULP)
 
 	mv	$(WORKDIR_WOODPULP)/gfw_plantations_woodpulp_harvest_ha_res.tif	$(WORKDIR_WOODPULP)/ha/gfw_plantations_woodpulp_ha.tif
 	mv	$(WORKDIR_WOODPULP)/gfw_plantations_woodpulp_prod_t_nd_res.tif	$(WORKDIR_WOODPULP)/prod/gfw_plantations_woodpulp_prod.tif

diff --git a/infrastructure/kubernetes/main.tf b/infrastructure/kubernetes/main.tf
@@ -67,6 +67,4 @@ module "environment" {
   data_import_container_registry_url = data.terraform_remote_state.core.outputs.data_import_container_registry_url
   api_env_vars                       = lookup(each.value, "api_env_vars", [])
   api_secrets                        = lookup(each.value, "api_secrets", [])
-  data_s3_access_key                 = var.data_s3_access_key
-  data_s3_secret_key                 = var.data_s3_secret_key
 }
diff --git a/infrastructure/kubernetes/modules/env/main.tf b/infrastructure/kubernetes/modules/env/main.tf
@@ -149,8 +149,6 @@ module "k8s_secrets" {
   allowed_account_id = var.allowed_account_id
   namespace          = var.environment
   gmaps_api_key      = var.gmaps_api_key
-  data_s3_access_key = var.data_s3_access_key
-  data_s3_secret_key = var.data_s3_secret_key
 
   depends_on = [
     module.k8s_namespace

diff --git a/infrastructure/kubernetes/modules/env/variables.tf b/infrastructure/kubernetes/modules/env/variables.tf
@@ -68,18 +68,6 @@ variable "data_import_arguments" {
   description = "Arguments to pass to the initial data import process"
 }
 
-variable "data_s3_access_key" {
-  type        = string
-  sensitive   = true
-  description = "Science AWS S3 bucket access key"
-}
-
-variable "data_s3_secret_key" {
-  type        = string
-  sensitive   = true
-  description = "Science AWS S3 bucket secret key"
-}
-
 variable "api_container_registry_url" {
   type        = string
   description = "URL for the API container registry"

diff --git a/infrastructure/kubernetes/modules/secrets/main.tf b/infrastructure/kubernetes/modules/secrets/main.tf
@@ -13,11 +13,6 @@ locals {
     jwt_secret    = random_password.jwt_secret_generator.result
     gmaps_api_key = var.gmaps_api_key
   }
-
-  data_secret_json = {
-    data_s3_access_key = var.data_s3_access_key
-    data_s3_secret_key = var.data_s3_secret_key
-  }
 }
 
 # JWT
@@ -116,11 +111,6 @@ resource "kubernetes_secret" "data_secret" {
     name      = "data"
     namespace = var.namespace
   }
-
-  data = {
-    DATA_S3_ACCESS_KEY      = local.data_secret_json.data_s3_access_key
-    DATA_S3_SECRET_KEY      = local.data_secret_json.data_s3_secret_key
-  }
 }
 
 

diff --git a/infrastructure/kubernetes/modules/secrets/variable.tf b/infrastructure/kubernetes/modules/secrets/variable.tf
@@ -28,15 +28,3 @@ variable "gmaps_api_key" {
   sensitive   = true
   description = "The Google Maps API key used for access to the geocoding API"
 }
-
-variable "data_s3_access_key" {
-  type        = string
-  sensitive   = true
-  description = "Science AWS S3 bucket access key"
-}
-
-variable "data_s3_secret_key" {
-  type        = string
-  sensitive   = true
-  description = "Science AWS S3 bucket secret key"
-}
diff --git a/infrastructure/kubernetes/variables.tf b/infrastructure/kubernetes/variables.tf
@@ -49,15 +49,3 @@ variable "mapbox_api_token" {
 variable "environments" {
   description = "A list of environments"
 }
-
-variable "data_s3_access_key" {
-  type        = string
-  sensitive   = true
-  description = "Science AWS S3 bucket access key"
-}
-
-variable "data_s3_secret_key" {
-  type        = string
-  sensitive   = true
-  description = "Science AWS S3 bucket secret key"
-}