Merge pull request #10 from DP6/main

build: 🏗️ terraform improvements and CI

.github/workflows/terraform.yml

@@ -0,0 +1,34 @@
+name: Terraform Validate
+
+on: ['push']
+
+jobs:
+  terraform-actions:
+    name: Workflow
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./terraform
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@master
+
+      - name: HashiCorp - Setup Terraform
+        uses: hashicorp/[email protected]
+        with:
+          terraform_version: 0.14.6
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+        continue-on-error: true
+
+      - name: Terraform Fmt
+        id: fmt
+        run: terraform fmt -check -diff
+        continue-on-error: true
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+        continue-on-error: false

terraform/external.tf

@@ -0,0 +1,8 @@
+#create detaflow metadata
+resource "null_resource" "bucket_megalista_metadata" {
+  provisioner "local-exec" {
+    command = "sh ./scripts/deploy_cloud.sh ${data.google_client_config.current.project} ${var.bucket_name} ${var.region}"
+  }
+
+  depends_on = [google_storage_bucket.my_storage]
+}

terraform/main.tf

@@ -1,52 +1,13 @@
-variable "bucket_name" {
-    type = string
-    description = "Google Cloud Storage Bucket to create"
-}
-
-variable "bq_ops_dataset" {
-    type = string
-    description = "Auxliary bigquery dataset for Megalista operations to create"
-}
-
-variable "developer_token" {
-    type = string
-    description = "Google Ads developer Token"
-}
-
-variable "client_id" {
-    type = string
-    description = "OAuth Client Id"
-}
-
-variable "client_secret" {
-    type = string
-    description = "OAuth Client Secret"
-}
-
-variable "access_token" {
-    type = string
-    description = "Access Token"
-}
-
-variable "refresh_token" {
-    type = string
-    description = "Refresh Token"
-}
-
-variable "setup_sheet_id" {
-    type = string
-    description = "Setup Sheet Id"
-}
-
 data "google_client_config" "current" {
 }
 
 data "google_client_openid_userinfo" "me" {
 }
 
 resource "google_bigquery_dataset" "dataset" {
-  dataset_id                  = "${var.bq_ops_dataset}"
-  location                    = "US"
+  dataset_id                  = var.bq_ops_dataset
+  location                    = var.location
+  description                 = "Auxliary bigquery dataset for Megalista operations to create"
   delete_contents_on_destroy = true
 }
 
@@ -65,17 +26,17 @@ locals {
         },
         "environment": {
             "tempLocation": "gs://${var.bucket_name}/tmp",
-            "zone": "us-central1-f"
+            "zone": "${var.zone}"
         }
     }
     EOF
 }
 
 resource "google_storage_bucket" "my_storage" {
   name          = var.bucket_name
-  location      = "US"
+  location      = var.location
   force_destroy = true
-  bucket_policy_only = true
+  uniform_bucket_level_access = true
 }
 
 resource "google_project_iam_member" "cloudscheduler-creator" {
@@ -160,13 +121,13 @@ resource "google_cloud_scheduler_job" "megalista_job" {
   name             = "megalista_job"
   description      = "Daily Runner for Megalista"
   schedule         = "0 0 * * *"
-  time_zone        = "America/Sao_Paulo"
+  time_zone        = local.time_zone
   attempt_deadline = "320s"
-  region           = "us-central1"
+  region           = var.region
 
   http_target {
     http_method = "POST"
-    uri         = "https://dataflow.googleapis.com/v1b3/projects/${data.google_client_config.current.project}/templates:launch?gcsPath=gs://${var.bucket_name}/templates/megalist"
+    uri         = "https://dataflow.googleapis.com/v1b3/projects/${data.google_client_config.current.project}/templates:launch?gcsPath=gs://${var.bucket_name}/templates/megalista"
     body        = base64encode(local.scheduler_body)
     oauth_token {
       service_account_email = google_service_account.sa.email

deploy_cloud.sh → terraform/scripts/deploy_cloud.sh

@@ -19,9 +19,14 @@ if [ $# != 3 ]; then
     exit 1
 fi
 
+echo "Move to megalist_dataflow folder"
+cd ..
 cd megalist_dataflow
-gcloud config set project $1
+echo "Configuration GCP project in gcloud"
+gcloud config set project "$1"
+echo "Build Dataflow metadata"
 python3 -m pip install --user -q -r requirements.txt
-python3 -m main --runner DataflowRunner --project $1 --gcp_project_id $1 --temp_location gs://$2/tmp/ --region $3 --setup_file ./setup.py --template_location gs://$2/templates/megalist --num_workers 1 --autoscaling_algorithm=NONE
-gsutil cp megalist_metadata gs://$2/templates/megalist_metadata
-cd ..
+python3 -m main --runner DataflowRunner --project "$1" --gcp_project_id "$1" --temp_location"gs://$2/tmp/" --region "$3" --setup_file ./setup.py --template_location "gs://$2/templates/megalista" --num_workers 1 --autoscaling_algorithm=NONE
+echo "Copy megalista_medata to bucket $2"
+gsutil cp megalist_metadata "gs://$2/templates/megalista_metadata"
+cd ..

terraform/variables.tf

@@ -0,0 +1,62 @@
+locals {
+  time_zone   = "America/Sao_Paulo"
+}
+
+variable "bucket_name" {
+    type = string
+    description = "Google Cloud Storage Bucket to create"
+}
+
+variable "bq_ops_dataset" {
+    type = string
+    description = "Auxliary bigquery dataset for Megalista operations to create"
+}
+
+variable "developer_token" {
+    type = string
+    description = "Google Ads developer Token"
+}
+
+variable "client_id" {
+    type = string
+    description = "OAuth Client Id"
+}
+
+variable "client_secret" {
+    type = string
+    description = "OAuth Client Secret"
+}
+
+variable "access_token" {
+    type = string
+    description = "Access Token"
+}
+
+variable "refresh_token" {
+    type = string
+    description = "Refresh Token"
+}
+
+variable "setup_sheet_id" {
+    type = string
+    description = "Setup Sheet Id"
+}
+
+variable "location" {
+  type        = string
+  description = "GCP location https://cloud.google.com/compute/docs/regions-zones?hl=pt-br default us"
+  default     = "us"
+}
+
+variable "region" {
+  type        = string
+  description = "GCP region https://cloud.google.com/compute/docs/regions-zones?hl=pt-br default us-central1"
+  default     = "us-central1"
+}
+
+variable "zone" {
+  type        = string
+  description = "GCP zone https://cloud.google.com/compute/docs/regions-zones?hl=pt-br default us-central1"
+  default     = "us-central1-f"
+}
+