GCP deployment - TF setup

Vizzuality · Apr 17, 2023 · dcb913b · dcb913b
1 parent d8b7da1
commit dcb913b
Show file tree

Hide file tree

Showing 77 changed files with 4,567 additions and 266 deletions.
diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml
@@ -1,12 +1,17 @@
 name: Publish Docker images
 
+env:
+  PROJECT_ID: ${{ secrets.GCP_PROJECT }}
+  REGION: ${{ secrets.GCP_REGION }}
+
 on:
   push:
     branches:
       - 'main'
       - 'staging'
       - 'test'
       - 'vcf'
+      - 'gcp'
       - 'tetrapack'
       - 'dev'
       - 'demo'
@@ -57,9 +62,14 @@ jobs:
     name: Push API Docker image to Docker Hub
     runs-on: ubuntu-20.04
     needs: wait_for_tests
+
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -72,6 +82,18 @@ jobs:
         id: ecr-login
         uses: aws-actions/amazon-ecr-login@v1
 
+      - name: 'Authenticate to Google Cloud'
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: 'projects/168301767246/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: '[email protected]'
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Authorize Docker push
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Extract branch name
         shell: bash
         run: |
@@ -85,23 +107,33 @@ jobs:
           ECR_REPOSITORY: api
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG api
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/api/main:${{ github.sha }} \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/api/main:latest \
+          api
 
-      - name: Push API Docker image to AWS ECR
+      - name: Push API Docker image to AWS and GCP
         env:
           ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }}
           ECR_REPOSITORY: api
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/api/main:${{ github.sha }}
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/api/main:latest
 
   push_client_to_registry:
     name: Push Client Docker image to Docker Hub
     runs-on: ubuntu-20.04
     needs: wait_for_tests
+
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -114,6 +146,18 @@ jobs:
         id: ecr-login
         uses: aws-actions/amazon-ecr-login@v1
 
+      - name: 'Authenticate to Google Cloud'
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: 'projects/168301767246/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: '[email protected]'
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Authorize Docker push
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Extract branch name
         shell: bash
         run: |
@@ -134,25 +178,34 @@ jobs:
           --build-arg NEXT_PUBLIC_API_URL=${{ secrets[format('NEXT_PUBLIC_API_URL_{0}', steps.extract_branch.outputs.branch-upper )] }} \
           --build-arg CYPRESS_USERNAME=${{ secrets.CYPRESS_USERNAME }} \
           --build-arg CYPRESS_PASSWORD=${{ secrets.CYPRESS_PASSWORD }} \
-          -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \
+          -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG  \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/client/main:${{ github.sha }} \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/client/main:latest \
           client
 
-      - name: Push Client Docker image to AWS ECR
+      - name: Push Client Docker image to AWS and GCP
         env:
           ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }}
           ECR_REPOSITORY: client
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/client/main:${{ github.sha }}
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/client/main:latest
 
 
   push_data_import_to_registry:
     name: Push Data Import Docker image to Docker Hub
     runs-on: ubuntu-20.04
     needs: wait_for_tests
+
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -165,6 +218,18 @@ jobs:
         id: ecr-login
         uses: aws-actions/amazon-ecr-login@v1
 
+      - name: 'Authenticate to Google Cloud'
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: 'projects/168301767246/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: '[email protected]'
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Authorize Docker push
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Extract branch name
         shell: bash
         run: |
@@ -178,23 +243,33 @@ jobs:
           ECR_REPOSITORY: data_import
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG data
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG  \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/data-import/main:${{ github.sha }} \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/data-import/main:latest \
+          data
 
-      - name: Push Data Import Docker image to AWS ECR
+      - name: Push Data Import Docker image to AWS and GCP
         env:
           ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }}
           ECR_REPOSITORY: data_import
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/data-import/main:${{ github.sha }}
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/data-import/main:latest
 
   push_tiler_to_registry:
-    name: Push Tiler Docker image to AWS ECR
+    name: Push Tiler Docker image to AWS and GCP
     runs-on: ubuntu-20.04
     needs: wait_for_tests
+
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -207,6 +282,18 @@ jobs:
         id: ecr-login
         uses: aws-actions/amazon-ecr-login@v1
 
+      - name: 'Authenticate to Google Cloud'
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: 'projects/168301767246/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: '[email protected]'
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Authorize Docker push
+        run: gcloud auth configure-docker europe-west1-docker.pkg.dev
+
       - name: Extract branch name
         shell: bash
         run: |
@@ -220,11 +307,17 @@ jobs:
           ECR_REPOSITORY: tiler
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG tiler
-      - name: Push Tiler Docker image to AWS ECR
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/tiler/main:${{ github.sha }} \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/tiler/main:latest \
+          tiler
+
+      - name: Push Tiler Docker image to AWS and GCP
         env:
           ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }}
           ECR_REPOSITORY: tiler
           IMAGE_TAG: ${{ steps.extract_branch.outputs.branch }}
         run: |
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/tiler/main:${{ github.sha }}
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/tiler/main:latest
diff --git a/.github/workflows/publish-marketing-site.yml b/.github/workflows/publish-marketing-site.yml
@@ -2,7 +2,6 @@ name: Publish marketing site image
 
 env:
   PROJECT_ID: ${{ secrets.GCP_PROJECT }}
-  SERVICE: 'marketing'
   REGION: ${{ secrets.GCP_REGION }}
 
 on:
@@ -12,6 +11,7 @@ on:
       - 'staging'
       - 'test'
       - 'vcf'
+      - 'gcp'
       - 'tetrapack'
       - 'dev'
       - 'demo'
@@ -63,15 +63,15 @@ jobs:
           --build-arg SENDGRID_API_KEY_SUBSCRIPTION=${{ secrets.MARKETING_SENDGRID_API_KEY_SUBSCRIPTION }} \
           --build-arg SENDGRID_API_KEY_CONTACT=${{ secrets.MARKETING_SENDGRID_API_KEY_CONTACT }} \
           --build-arg NEXT_PUBLIC_GOOGLE_ANALYTICS=${{ secrets.MARKETING_NEXT_PUBLIC_GOOGLE_ANALYTICS }} \
-          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}/main:${{ github.sha }} \
-          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}/main:latest ./marketing
-          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}/main:${{ github.sha }}
-          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}/main:latest
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/marketing/main:${{ github.sha }} \
+          -t europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/marketing/main:latest ./marketing
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/marketing/main:${{ github.sha }}
+          docker push europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/marketing/main:latest
 
       - name: Deploy to Cloud Run
         run: |-
-          gcloud run deploy ${{ env.SERVICE }} \
+          gcloud run deploy marketing \
             --region ${{ env.REGION }} \
-            --image europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}/main:latest \
+            --image europe-west1-docker.pkg.dev/${{ env.PROJECT_ID }}/marketing/main:latest \
             --platform "managed" \
             --quiet
diff --git a/data/h3_data_importer/Makefile b/data/h3_data_importer/Makefile
@@ -19,6 +19,9 @@ WORKDIR_HDI=data/contextual/hdi
 WORKDIR_GHG=data/forest_ghg
 WORKDIR_WOODPULP=data/woodpulp
 
+export AWS_ACCESS_KEY_ID = $(DATA_S3_ACCESS_KEY)
+export AWS_SECRET_ACCESS_KEY = $(DATA_S3_SECRET_KEY)
+
 all:
 	@aws s3 ls $(AWS_S3_BUCKET_URL) 2>&1 > /dev/null; \
 	if [ $$? -ne 0 ]; \

diff --git a/infrastructure/base/aws.tf b/infrastructure/base/aws.tf
@@ -145,3 +145,34 @@ resource "aws_iam_role_policy_attachment" "raw_s3_rw_access_attachment" {
   role       = module.eks.node_role.name
   policy_arn = aws_iam_policy.raw_s3_rw_access.arn
 }
+
+resource "aws_iam_user" "raw_s3_reader" {
+  name = "ReadAccessToRawDataS3Bucket"
+}
+
+resource "aws_iam_policy" "raw_s3_read_access" {
+  name        = "ReadAccessToRawDataS3Bucket"
+  description = "Read access to the raw data S3 bucket"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        "Action" : [
+          "s3:Get*",
+          "s3:List*",
+        ],
+        Effect = "Allow"
+        Resource = [
+          module.s3_bucket.bucket_arn,
+          "${module.s3_bucket.bucket_arn}/*",
+        ]
+      },
+    ]
+  })
+}
+
+resource "aws_iam_user_policy_attachment" "raw_s3_rw_access_attachment" {
+  user       = aws_iam_user.raw_s3_reader.name
+  policy_arn = aws_iam_policy.raw_s3_read_access.arn
+}
diff --git a/infrastructure/base/gcp.tf b/infrastructure/base/gcp.tf
@@ -28,6 +28,38 @@ module "marketing_gcr" {
   service_account = module.workload_identity.service_account
 }
 
+module "client_gcr" {
+  source          = "./modules/gcp/gcr"
+  project_id      = var.gcp_project_id
+  region          = var.gcp_region
+  name            = "client"
+  service_account = module.gke.node_service_account
+}
+
+module "api_gcr" {
+  source          = "./modules/gcp/gcr"
+  project_id      = var.gcp_project_id
+  region          = var.gcp_region
+  name            = "api"
+  service_account = module.gke.node_service_account
+}
+
+module "tiler_gcr" {
+  source          = "./modules/gcp/gcr"
+  project_id      = var.gcp_project_id
+  region          = var.gcp_region
+  name            = "tiler"
+  service_account = module.gke.node_service_account
+}
+
+module "data_import_gcr" {
+  source          = "./modules/gcp/gcr"
+  project_id      = var.gcp_project_id
+  region          = var.gcp_region
+  name            = "data-import"
+  service_account = module.gke.node_service_account
+}
+
 module "load_balancer" {
   source                  = "./modules/gcp/load-balancer"
   region                  = var.gcp_region
@@ -42,3 +74,14 @@ module "workload_identity" {
   project_id = var.gcp_project_id
 }
 
+
+module "gke" {
+  source         = "./modules/gcp/gke"
+  cluster_name   = var.project_name
+  node_pool_name = "default-pool"
+  zone           = var.gcp_zone
+  region         = var.gcp_region
+  project        = var.gcp_project_id
+  network        = module.network.network_name
+  subnetwork     = module.network.subnetwork_name
+}
diff --git a/infrastructure/base/modules/gcp/gcr/outputs.tf b/infrastructure/base/modules/gcp/gcr/outputs.tf
@@ -0,0 +1,3 @@
+output "artifact_registry_repository_url" {
+  value = "${google_artifact_registry_repository.repository.location}-docker.pkg.dev/${google_artifact_registry_repository.repository.project}/${google_artifact_registry_repository.repository.name}/main"
+}