diff --git a/data/h3_data_importer/Makefile b/data/h3_data_importer/Makefile index fb7186b418..7d339999c1 100644 --- a/data/h3_data_importer/Makefile +++ b/data/h3_data_importer/Makefile @@ -6,6 +6,7 @@ ######################### PARALLELIZATION_FACTOR=10 +AWS_S3_BUCKET_URL=s3://landgriffon-raw-data WORKDIR_MAPSPAM=data/mapspam WORKDIR_GLW3=data/glw3 WORKDIR_WATER_FOOTPRINT=data/waterFootprint @@ -18,16 +19,16 @@ WORKDIR_HDI=data/contextual/hdi WORKDIR_GHG=data/forest_ghg WORKDIR_WOODPULP=data/woodpulp -export AWS_ACCESS_KEY_ID = $(DATA_S3_ACCESS_KEY) -export AWS_SECRET_ACCESS_KEY = $(DATA_S3_SECRET_KEY) - all: -ifeq ($(DATA_S3_ACCESS_KEY),) - $(error AWS S3 credentials are not defined in env vars. Import will not run) -else + @aws s3 ls $(AWS_S3_BUCKET_URL) 2>&1 > /dev/null; \ + if [ $$? -ne 0 ]; \ + then \ + echo "Error: Cannot access the expected AWS S3 bucket."; \ + false; \ + fi + @echo "AWS S3 bucket access confirmed, proceeding with import..." make clean make -j 2 crop indicators -endif crop: make -j 2 convert-mapspam-crop-production convert-mapspam-crop-harvest convert-glw3-livestock convert-woodpulp @@ -436,7 +437,7 @@ convert-aboveGroundBiomass: preprocess-aboveGroundBiomass download-satDeforestation: mkdir -p $(WORKDIR_SATDEFORESTATION) - aws s3 sync s3://landgriffon-raw-data/processed/satelligence $(WORKDIR_SATDEFORESTATION) + aws s3 sync $(AWS_S3_BUCKET_URL)/processed/satelligence $(WORKDIR_SATDEFORESTATION) mkdir -p $(WORKDIR_SATDEFORESTATION)/mask mkdir -p $(WORKDIR_SATDEFORESTATION)/risk @@ -456,7 +457,7 @@ download-woodpulp: mkdir -p $(WORKDIR_WOODPULP)/ha mkdir -p $(WORKDIR_WOODPULP)/prod - aws s3 sync s3://landgriffon-raw-data/processed/woodpulp/ $(WORKDIR_WOODPULP) + aws s3 sync $(AWS_S3_BUCKET_URL)/processed/woodpulp/ $(WORKDIR_WOODPULP) mv $(WORKDIR_WOODPULP)/gfw_plantations_woodpulp_harvest_ha_res.tif $(WORKDIR_WOODPULP)/ha/gfw_plantations_woodpulp_ha.tif mv $(WORKDIR_WOODPULP)/gfw_plantations_woodpulp_prod_t_nd_res.tif $(WORKDIR_WOODPULP)/prod/gfw_plantations_woodpulp_prod.tif diff --git a/infrastructure/kubernetes/main.tf b/infrastructure/kubernetes/main.tf index f7f853d226..ce0e3fc1ba 100644 --- a/infrastructure/kubernetes/main.tf +++ b/infrastructure/kubernetes/main.tf @@ -67,6 +67,4 @@ module "environment" { data_import_container_registry_url = data.terraform_remote_state.core.outputs.data_import_container_registry_url api_env_vars = lookup(each.value, "api_env_vars", []) api_secrets = lookup(each.value, "api_secrets", []) - data_s3_access_key = var.data_s3_access_key - data_s3_secret_key = var.data_s3_secret_key } diff --git a/infrastructure/kubernetes/modules/env/main.tf b/infrastructure/kubernetes/modules/env/main.tf index c401ac4af9..ac05769289 100644 --- a/infrastructure/kubernetes/modules/env/main.tf +++ b/infrastructure/kubernetes/modules/env/main.tf @@ -149,8 +149,6 @@ module "k8s_secrets" { allowed_account_id = var.allowed_account_id namespace = var.environment gmaps_api_key = var.gmaps_api_key - data_s3_access_key = var.data_s3_access_key - data_s3_secret_key = var.data_s3_secret_key depends_on = [ module.k8s_namespace diff --git a/infrastructure/kubernetes/modules/env/variables.tf b/infrastructure/kubernetes/modules/env/variables.tf index 056ae24f9e..2daa46615e 100644 --- a/infrastructure/kubernetes/modules/env/variables.tf +++ b/infrastructure/kubernetes/modules/env/variables.tf @@ -68,18 +68,6 @@ variable "data_import_arguments" { description = "Arguments to pass to the initial data import process" } -variable "data_s3_access_key" { - type = string - sensitive = true - description = "Science AWS S3 bucket access key" -} - -variable "data_s3_secret_key" { - type = string - sensitive = true - description = "Science AWS S3 bucket secret key" -} - variable "api_container_registry_url" { type = string description = "URL for the API container registry" diff --git a/infrastructure/kubernetes/modules/secrets/main.tf b/infrastructure/kubernetes/modules/secrets/main.tf index 6472d69ede..09377c0364 100644 --- a/infrastructure/kubernetes/modules/secrets/main.tf +++ b/infrastructure/kubernetes/modules/secrets/main.tf @@ -13,11 +13,6 @@ locals { jwt_secret = random_password.jwt_secret_generator.result gmaps_api_key = var.gmaps_api_key } - - data_secret_json = { - data_s3_access_key = var.data_s3_access_key - data_s3_secret_key = var.data_s3_secret_key - } } # JWT @@ -116,11 +111,6 @@ resource "kubernetes_secret" "data_secret" { name = "data" namespace = var.namespace } - - data = { - DATA_S3_ACCESS_KEY = local.data_secret_json.data_s3_access_key - DATA_S3_SECRET_KEY = local.data_secret_json.data_s3_secret_key - } } diff --git a/infrastructure/kubernetes/modules/secrets/variable.tf b/infrastructure/kubernetes/modules/secrets/variable.tf index 61bd24052c..294c38f5bd 100644 --- a/infrastructure/kubernetes/modules/secrets/variable.tf +++ b/infrastructure/kubernetes/modules/secrets/variable.tf @@ -28,15 +28,3 @@ variable "gmaps_api_key" { sensitive = true description = "The Google Maps API key used for access to the geocoding API" } - -variable "data_s3_access_key" { - type = string - sensitive = true - description = "Science AWS S3 bucket access key" -} - -variable "data_s3_secret_key" { - type = string - sensitive = true - description = "Science AWS S3 bucket secret key" -} diff --git a/infrastructure/kubernetes/variables.tf b/infrastructure/kubernetes/variables.tf index 52576b52b2..7c9cbecebe 100644 --- a/infrastructure/kubernetes/variables.tf +++ b/infrastructure/kubernetes/variables.tf @@ -49,15 +49,3 @@ variable "mapbox_api_token" { variable "environments" { description = "A list of environments" } - -variable "data_s3_access_key" { - type = string - sensitive = true - description = "Science AWS S3 bucket access key" -} - -variable "data_s3_secret_key" { - type = string - sensitive = true - description = "Science AWS S3 bucket secret key" -}