From ac18ce9911fbe0c71bd6349c63d76b469f87ede9 Mon Sep 17 00:00:00 2001 From: Elie CHARRA Date: Tue, 17 Dec 2024 17:24:21 +0100 Subject: [PATCH] feat: add external workers (#20) * feat: add external workers This adds a new input `enable_external_workers` that replaces the old `create_compute_address_for_mqtt` that was incomplete. If this flag is enabled, we create two additional IPs and a DNS zone to be able to resolve the same mqtt endpoint address from both inside and outside of the cluster. It also create a new artifact repo and enable public read access on it. We use the public repo as the exported launcher image variable. I moved subnetwork in network module. * fixup! feat: add external workers * fixup! feat: add external workers --- README.md | 8 ++++-- main.tf | 26 ++++++++++++++--- modules/artifacts/main.tf | 22 +++++++++++++++ modules/artifacts/outputs.tf | 15 ++++++++-- modules/artifacts/variables.tf | 4 +++ modules/dns/main.tf | 51 ++++++++++++++++++++++++++++++++++ modules/dns/outputs.tf | 4 +++ modules/dns/variables.tf | 35 +++++++++++++++++++++++ modules/gke/main.tf | 6 ++-- modules/gke/mqtt.tf | 17 ------------ modules/gke/network.tf | 25 ++--------------- modules/gke/outputs.tf | 20 ------------- modules/gke/variables.tf | 27 ++++++++---------- modules/network/main.tf | 43 ++++++++++++++++++++++++++++ modules/network/outputs.tf | 32 +++++++++++++++++++++ modules/network/variables.tf | 24 ++++++++++++++++ outputs.tf | 22 +++++++++------ variables.tf | 4 +-- 18 files changed, 287 insertions(+), 98 deletions(-) create mode 100644 modules/dns/main.tf create mode 100644 modules/dns/outputs.tf create mode 100644 modules/dns/variables.tf delete mode 100644 modules/gke/mqtt.tf diff --git a/README.md b/README.md index 1985e61..f8c5dbc 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The module creates: - a compute subnetwork for the GKE cluster - Artifact repository - a Google Artifact Registry repository for storing Docker images + - a PUBLIC Google Artifact Registry repository for storing Docker images for workers (if external workers are enabled) - Database resources - a Postgres Cloud SQL instance - Storage resources @@ -41,7 +42,7 @@ The module creates: ### Inputs | Name | Description | Type | Default | Required | -| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | --------------------- | -------- | +|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------- |-----------------------| -------- | | region | The region in which the resources will be created. | string | - | yes | | project | The ID of the project in which the resources will be created. | string | - | yes | | website_domain | The domain under which the Spacelift instance will be hosted. This is used for the CORS rules of one of the buckets. | string | - | yes | @@ -55,6 +56,7 @@ The module creates: | ip_cidr_range | The IP CIDR range for the subnetwork used by the GKE cluster | string | 10.0.0.0/16 | no | | secondary_ip_range_for_services | The secondary IP range for the subnetwork used by the GKE cluster. This range is used for services | string | 192.168.16.0/22 | no | | secondary_ip_range_for_pods | The secondary IP range for the subnetwork used by the GKE cluster. This range is used for pods | string | 192.168.0.0/20 | no | +| enable_external_workers | Switch this to true if you want to run workers from outside of the GCP infrastructure. | string | false | no | ### Outputs @@ -72,8 +74,8 @@ The module creates: | gke_cluster_name | Name of the GKE cluster. | | gke_public_v4_address | Public IPv4 address of the GKE cluster. | | gke_public_v6_address | Public IPv6 address of the GKE cluster. | -| mqtt_ipv4_address | IPv4 address of the MQTT service. It's null if create_compute_address_for_mqtt is false. It's only useful in case the workerpool is outside the GKE cluster. | -| mqtt_ipv6_address | IPv6 address of the MQTT service. It's null if create_compute_address_for_mqtt is false. It's only useful in case the workerpool is outside the GKE cluster. | +| mqtt_ipv4_address | IPv4 address of the MQTT service. It's null if enable_external_workers is false. It's only useful in case the workerpool is outside the GKE cluster. | +| mqtt_ipv6_address | IPv6 address of the MQTT service. It's null if enable_external_workers is false. It's only useful in case the workerpool is outside the GKE cluster. | | artifact_repository_url | URL of the Docker artifact repository. | | db_name | Name of the database. | | db_root_password | Database root password | diff --git a/main.tf b/main.tf index 01b79bf..037ab76 100644 --- a/main.tf +++ b/main.tf @@ -14,12 +14,20 @@ module "artifacts" { source = "./modules/artifacts" depends_on = [module.iam] seed = random_id.seed.hex + + enable_external_workers = var.enable_external_workers } module "network" { source = "./modules/network" depends_on = [module.iam] seed = random_id.seed.hex + + enable_external_workers = var.enable_external_workers + ip_cidr_range = var.ip_cidr_range + secondary_ip_range_for_pods = var.secondary_ip_range_for_pods + secondary_ip_range_for_services = var.secondary_ip_range_for_services + region = var.region } module "gke" { @@ -29,15 +37,14 @@ module "gke" { app_service_account_name = var.app_service_account_name backend_service_account_id = module.iam.backend_service_account_id compute_network_id = module.network.network_id + subnetwork = module.network.subnetwork + pods_ip_range_name = module.network.pods_ip_range_name + services_ip_range_name = module.network.services_ip_range_name compute_network_name = module.network.network_name - create_compute_address_for_mqtt = var.create_compute_address_for_mqtt gke_service_account_email = module.iam.gke_service_account_email - ip_cidr_range = var.ip_cidr_range k8s_namespace = var.k8s_namespace project = var.project region = var.region - secondary_ip_range_for_pods = var.secondary_ip_range_for_pods - secondary_ip_range_for_services = var.secondary_ip_range_for_services } module "db" { @@ -62,3 +69,14 @@ module "storage" { project = var.project region = var.region } + +module "dns" { + source = "./modules/dns" + seed = random_id.seed.hex + + enable_external_workers = var.enable_external_workers + website_domain = var.website_domain + compute_network_id = module.network.network_id + gke_public_v4_address = module.network.gke_public_v4_address + gke_public_v6_address = module.network.gke_public_v6_address +} diff --git a/modules/artifacts/main.tf b/modules/artifacts/main.tf index 7f5fa20..261a7dd 100644 --- a/modules/artifacts/main.tf +++ b/modules/artifacts/main.tf @@ -11,3 +11,25 @@ resource "google_artifact_registry_repository" "spacelift" { } } } + +resource "google_artifact_registry_repository" "spacelift-public" { + count = var.enable_external_workers ? 1 : 0 + repository_id = "spacelift-public-${var.seed}" + format = "DOCKER" + description = "This repository contains the public images for Spacelift" + + cleanup_policies { + id = "keep-10-images" + action = "KEEP" + most_recent_versions { + keep_count = 10 + } + } +} + +resource "google_artifact_registry_repository_iam_binding" "public" { + count = var.enable_external_workers ? 1 : 0 + members = ["allUsers"] + repository = google_artifact_registry_repository.spacelift-public[0].id + role = "roles/artifactregistry.reader" +} diff --git a/modules/artifacts/outputs.tf b/modules/artifacts/outputs.tf index 43a61db..8cd733b 100644 --- a/modules/artifacts/outputs.tf +++ b/modules/artifacts/outputs.tf @@ -1,9 +1,20 @@ +locals { + repository_domain = "${google_artifact_registry_repository.spacelift.location}-docker.pkg.dev" + repository_url = "${local.repository_domain}/${google_artifact_registry_repository.spacelift.project}/${google_artifact_registry_repository.spacelift.repository_id}" + public_repository_url = var.enable_external_workers ? "${local.repository_domain}/${google_artifact_registry_repository.spacelift-public[0].project}/${google_artifact_registry_repository.spacelift-public[0].repository_id}" : local.repository_url +} + output "repository_domain" { - value = "${google_artifact_registry_repository.spacelift.location}-docker.pkg.dev/" + value = local.repository_domain description = "The domain of the Docker repository" } output "repository_url" { - value = "${google_artifact_registry_repository.spacelift.location}-docker.pkg.dev/${google_artifact_registry_repository.spacelift.project}/${google_artifact_registry_repository.spacelift.repository_id}" + value = local.repository_url description = "The URL of the Docker repository" } + +output "launcher_repository_url" { + value = local.public_repository_url + description = "The URL of the public Docker repository" +} diff --git a/modules/artifacts/variables.tf b/modules/artifacts/variables.tf index f651379..a26c37f 100644 --- a/modules/artifacts/variables.tf +++ b/modules/artifacts/variables.tf @@ -1,3 +1,7 @@ variable "seed" { type = string } + +variable "enable_external_workers" { + type = bool +} diff --git a/modules/dns/main.tf b/modules/dns/main.tf new file mode 100644 index 0000000..0fbfeae --- /dev/null +++ b/modules/dns/main.tf @@ -0,0 +1,51 @@ +locals { + dns_name = join(".", slice(split(".", var.website_domain), length(split(".", var.website_domain))-2, length(split(".", var.website_domain)))) + count = var.enable_external_workers ? 1 : 0 +} + +resource "google_dns_managed_zone" "main" { + count = local.count + name = "${replace(local.dns_name, ".", "-")}-${var.seed}" + dns_name = "${local.dns_name}." + + visibility = "private" + + private_visibility_config { + networks { + network_url = var.compute_network_id + } + } +} + +resource "google_dns_record_set" "CNAME_mqtt" { + count = local.count + managed_zone = google_dns_managed_zone.main[0].name + + name = "${var.mqtt_subdomain}.${var.website_domain}." + type = "CNAME" + ttl = 300 + + rrdatas = [var.mqtt_service_alias] +} + +resource "google_dns_record_set" "A_website_domain" { + count = local.count + managed_zone = google_dns_managed_zone.main[0].name + + name = "${var.website_domain}." + type = "A" + ttl = 300 + + rrdatas = [var.gke_public_v4_address] +} + +resource "google_dns_record_set" "AAAA_website_domain" { + count = local.count + managed_zone = google_dns_managed_zone.main[0].name + + name = "${var.website_domain}." + type = "AAAA" + ttl = 300 + + rrdatas = [var.gke_public_v6_address] +} diff --git a/modules/dns/outputs.tf b/modules/dns/outputs.tf new file mode 100644 index 0000000..53b4bb7 --- /dev/null +++ b/modules/dns/outputs.tf @@ -0,0 +1,4 @@ +output "mqtt_endpoint" { + value = var.enable_external_workers ? trimsuffix(google_dns_record_set.CNAME_mqtt[0].name, ".") : var.mqtt_service_alias + description = "Address of the MQTT endpoint." +} diff --git a/modules/dns/variables.tf b/modules/dns/variables.tf new file mode 100644 index 0000000..de4f5ad --- /dev/null +++ b/modules/dns/variables.tf @@ -0,0 +1,35 @@ +variable "seed" { + type = string +} + +variable "enable_external_workers" { + type = bool +} + +variable "website_domain" { + type = string + description = "Domain name for the Spacelift frontend without protocol (e.g. spacelift.mycompany.com)." +} + +variable "compute_network_id" { + type = string + description = "The ID of the network to create the GKE cluster in" +} + +variable "gke_public_v4_address" { + type = string +} + +variable "gke_public_v6_address" { + type = string +} + +variable "mqtt_subdomain" { + type = string + default = "mqtt" +} + +variable "mqtt_service_alias" { + type = string + default = "spacelift-mqtt.spacelift.svc.cluster.local." +} diff --git a/modules/gke/main.tf b/modules/gke/main.tf index ae86594..26127e0 100644 --- a/modules/gke/main.tf +++ b/modules/gke/main.tf @@ -15,7 +15,7 @@ resource "google_container_cluster" "spacelift" { } network = var.compute_network_id - subnetwork = google_compute_subnetwork.default.self_link + subnetwork = var.subnetwork.self_link private_cluster_config { # This prevents nodes to be assigned any external IP that can be routed on the internet @@ -24,8 +24,8 @@ resource "google_container_cluster" "spacelift" { ip_allocation_policy { stack_type = "IPV4_IPV6" - services_secondary_range_name = google_compute_subnetwork.default.secondary_ip_range[0].range_name - cluster_secondary_range_name = google_compute_subnetwork.default.secondary_ip_range[1].range_name + services_secondary_range_name = var.services_ip_range_name + cluster_secondary_range_name = var.pods_ip_range_name } # Set `deletion_protection` to `true` will ensure that one cannot diff --git a/modules/gke/mqtt.tf b/modules/gke/mqtt.tf deleted file mode 100644 index c3b1ec0..0000000 --- a/modules/gke/mqtt.tf +++ /dev/null @@ -1,17 +0,0 @@ -resource "google_compute_address" "gke-mqtt-v4" { - count = var.create_compute_address_for_mqtt ? 1 : 0 - - name = "gke-mqtt-v4-${var.seed}" - address_type = "EXTERNAL" - ip_version = "IPV4" -} - -resource "google_compute_address" "gke-mqtt-v6" { - count = var.create_compute_address_for_mqtt ? 1 : 0 - - name = "gke-mqtt-v6-${var.seed}" - address_type = "EXTERNAL" - ip_version = "IPV6" - ipv6_endpoint_type = "NETLB" - subnetwork = google_compute_subnetwork.default.id -} diff --git a/modules/gke/network.tf b/modules/gke/network.tf index d8db6f3..bd2973d 100644 --- a/modules/gke/network.tf +++ b/modules/gke/network.tf @@ -1,24 +1,3 @@ -resource "google_compute_subnetwork" "default" { - name = "spacelift-gke-cluster-subnetwork" - - ip_cidr_range = var.ip_cidr_range - region = var.region - - stack_type = "IPV4_IPV6" - ipv6_access_type = "EXTERNAL" - - network = var.compute_network_id - secondary_ip_range { - range_name = "services-range" - ip_cidr_range = var.secondary_ip_range_for_services - } - - secondary_ip_range { - range_name = "pod-ranges" - ip_cidr_range = var.secondary_ip_range_for_pods - } -} - # This module create a Cloud router & Cloud NAT is used to allow outbound traffic from k8s pods # Only SNAT is allowed here, this should not be used for routing incoming traffic to pods. module "gke-router" { @@ -34,10 +13,10 @@ module "gke-router" { source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" subnetworks = [ { - name = google_compute_subnetwork.default.id + name = var.subnetwork.id source_ip_ranges_to_nat = ["PRIMARY_IP_RANGE", "LIST_OF_SECONDARY_IP_RANGES"] secondary_ip_range_names = [ - google_compute_subnetwork.default.secondary_ip_range[1].range_name, + var.pods_ip_range_name, ] } ] diff --git a/modules/gke/outputs.tf b/modules/gke/outputs.tf index 315b317..d47ce19 100644 --- a/modules/gke/outputs.tf +++ b/modules/gke/outputs.tf @@ -1,24 +1,4 @@ -output "gke_subnetwork_id" { - value = google_compute_subnetwork.default.id - description = "The ID of the subnetwork that the GKE cluster was created in" -} - -output "gke_subnetwork_name" { - value = google_compute_subnetwork.default.name - description = "The name of the subnetwork that the GKE cluster was created in" -} - output "gke_cluster_name" { value = google_container_cluster.spacelift.name description = "The name of the GKE cluster" } - -output "mqtt_ipv4_address" { - value = var.create_compute_address_for_mqtt ? google_compute_address.gke-mqtt-v4[0].address : null - description = "The IPv4 address of the MQTT service" -} - -output "mqtt_ipv6_address" { - value = var.create_compute_address_for_mqtt ? google_compute_address.gke-mqtt-v6[0].address : null - description = "The IPv6 address of the MQTT service" -} diff --git a/modules/gke/variables.tf b/modules/gke/variables.tf index 5e2ed6a..b36b4e3 100644 --- a/modules/gke/variables.tf +++ b/modules/gke/variables.tf @@ -23,29 +23,24 @@ variable "compute_network_id" { description = "The ID of the network to create the GKE cluster in" } -variable "compute_network_name" { - type = string - description = "The name of the network to create the GKE cluster in" +variable "subnetwork" { + type = object({ + id = string + self_link = string + }) } -variable "create_compute_address_for_mqtt" { - type = bool - description = "Whether to create a compute address for MQTT. It is meant to be used by Service of type LoadBalancer from the GKE cluster to expose the embedded MQTT server to the world. This is only required if you want to run worker outside of the GKE cluster." -} - -variable "ip_cidr_range" { - type = string - description = "The IP CIDR range for the GKE cluster" +variable "services_ip_range_name" { + type = string } -variable "secondary_ip_range_for_services" { - type = string - description = "The secondary IP range for services" +variable "pods_ip_range_name" { + type = string } -variable "secondary_ip_range_for_pods" { +variable "compute_network_name" { type = string - description = "The secondary IP range for pods" + description = "The name of the network to create the GKE cluster in" } variable "gke_service_account_email" { diff --git a/modules/network/main.tf b/modules/network/main.tf index 17f17da..5908ed7 100644 --- a/modules/network/main.tf +++ b/modules/network/main.tf @@ -4,6 +4,27 @@ resource "google_compute_network" "default" { enable_ula_internal_ipv6 = true } +resource "google_compute_subnetwork" "default" { + name = "spacelift-gke-cluster-subnetwork" + + ip_cidr_range = var.ip_cidr_range + region = var.region + + stack_type = "IPV4_IPV6" + ipv6_access_type = "EXTERNAL" + + network = google_compute_network.default.id + secondary_ip_range { + range_name = "services-range" + ip_cidr_range = var.secondary_ip_range_for_services + } + + secondary_ip_range { + range_name = "pod-ranges" + ip_cidr_range = var.secondary_ip_range_for_pods + } +} + # This public v4 address is meant to be used by Ingresses from the GKE cluster to expose services to the world. resource "google_compute_global_address" "gke-public-v4" { name = "spacelift-gke-public-v4-${var.seed}" @@ -17,3 +38,25 @@ resource "google_compute_global_address" "gke-public-v6" { address_type = "EXTERNAL" ip_version = "IPV6" } + +# This public v4 address is meant to be used by Service of type LoadBalancer from the GKE cluster +# to expose the embedded MQTT server to the world. This is only required if we want to run worker outside of the +# GKE cluster. +resource "google_compute_address" "mqtt-v4" { + count = var.enable_external_workers ? 1 : 0 + name = "gke-mqtt-v4-${var.seed}" + address_type = "EXTERNAL" + ip_version = "IPV4" +} + +# This public v6 address is meant to be used by Service of type LoadBalancer from the GKE cluster +# to expose the embedded MQTT server to the world. This is only required if we want to run worker outside of the +# GKE cluster. +resource "google_compute_address" "mqtt-v6" { + count = var.enable_external_workers ? 1 : 0 + name = "gke-mqtt-v6-${var.seed}" + address_type = "EXTERNAL" + ip_version = "IPV6" + ipv6_endpoint_type = "NETLB" + subnetwork = google_compute_subnetwork.default.id +} diff --git a/modules/network/outputs.tf b/modules/network/outputs.tf index 8db5925..5b3ad66 100644 --- a/modules/network/outputs.tf +++ b/modules/network/outputs.tf @@ -3,6 +3,10 @@ output "network_id" { description = "ID of the Compute network" } +output "subnetwork" { + value = google_compute_subnetwork.default +} + output "network_name" { value = google_compute_network.default.name description = "Name of the Compute network" @@ -32,3 +36,31 @@ output "gke_public_v6_name" { value = google_compute_global_address.gke-public-v6.name description = "Public IPv4 address for GKE Ingresses" } + +output "mqtt_v4_address" { + value = length(google_compute_address.mqtt-v4) == 1 ? google_compute_address.mqtt-v4[0].address : "" + description = "Public IPv4 address for MQTT traffic." +} + +output "mqtt_v4_name" { + value = length(google_compute_address.mqtt-v4) == 1 ? google_compute_address.mqtt-v4[0].name : "" + description = "Public IPv4 address for MQTT traffic." +} + +output "mqtt_v6_address" { + value = length(google_compute_address.mqtt-v6) == 1 ? google_compute_address.mqtt-v6[0].address : "" + description = "Public IPv6 address for MQTT traffic." +} + +output "mqtt_v6_name" { + value = length(google_compute_address.mqtt-v6) == 1 ? google_compute_address.mqtt-v6[0].name : "" + description = "Public IPv6 address for MQTT traffic." +} + +output "services_ip_range_name" { + value = google_compute_subnetwork.default.secondary_ip_range[0].range_name +} + +output "pods_ip_range_name" { + value = google_compute_subnetwork.default.secondary_ip_range[1].range_name +} diff --git a/modules/network/variables.tf b/modules/network/variables.tf index f651379..61d32f1 100644 --- a/modules/network/variables.tf +++ b/modules/network/variables.tf @@ -1,3 +1,27 @@ variable "seed" { type = string } + +variable "enable_external_workers" { + type = bool +} + +variable "region" { + type = string + description = "The region to create the GKE cluster in" +} + +variable "ip_cidr_range" { + type = string + description = "The IP CIDR range for the GKE cluster" +} + +variable "secondary_ip_range_for_services" { + type = string + description = "The secondary IP range for services" +} + +variable "secondary_ip_range_for_pods" { + type = string + description = "The secondary IP range for pods" +} diff --git a/outputs.tf b/outputs.tf index e41cfd1..c5d950f 100644 --- a/outputs.tf +++ b/outputs.tf @@ -38,12 +38,12 @@ output "network_link" { } output "gke_subnetwork_id" { - value = module.gke.gke_subnetwork_id + value = module.network.subnetwork.id description = "Subnetwork ID of the GKE cluster" } output "gke_subnetwork_name" { - value = module.gke.gke_subnetwork_name + value = module.network.subnetwork.name description = "Subnetwork name of the GKE cluster" } @@ -63,13 +63,13 @@ output "gke_public_v6_address" { } output "mqtt_ipv4_address" { - value = module.gke.mqtt_ipv4_address - description = "The IPv4 address of the MQTT service. It is empty if 'create_compute_address_for_mqtt' is set to false." + value = module.network.mqtt_v4_address + description = "The IPv4 address of the MQTT service. It is empty if 'enable_external_workers' is set to false." } output "mqtt_ipv6_address" { - value = module.gke.mqtt_ipv6_address - description = "The IPv6 address of the MQTT service. It is empty if 'create_compute_address_for_mqtt' is set to false." + value = module.network.mqtt_v6_address + description = "The IPv6 address of the MQTT service. It is empty if 'enable_external_workers' is set to false." } ### Artifact store ### @@ -171,13 +171,19 @@ output "shell" { # Network PUBLIC_IP_NAME: module.network.gke_public_v4_name, + PUBLIC_IP_ADDRESS: module.network.gke_public_v4_address, PUBLIC_IPV6_NAME: module.network.gke_public_v6_name, + PUBLIC_IPV6_ADDRESS: module.network.gke_public_v6_address, + MQTT_IP_NAME: module.network.mqtt_v4_name, + MQTT_IP_ADDRESS: module.network.mqtt_v4_address, + MQTT_IPV6_NAME: module.network.mqtt_v6_name, + MQTT_IPV6_ADDRESS: module.network.mqtt_v6_address, + MQTT_BROKER_ENDPOINT: module.dns.mqtt_endpoint, # Artifacts ARTIFACT_REGISTRY_DOMAIN: module.artifacts.repository_domain, - ARTIFACT_REGISTRY_REPOSITORY: module.artifacts.repository_url, BACKEND_IMAGE: "${module.artifacts.repository_url}/spacelift-backend", - LAUNCHER_IMAGE: "${module.artifacts.repository_url}/spacelift-launcher" + LAUNCHER_IMAGE: "${module.artifacts.launcher_repository_url}/spacelift-launcher" # Buckets OBJECT_STORAGE_BUCKET_DELIVERIES=module.storage.deliveries_bucket, diff --git a/variables.tf b/variables.tf index 71664a9..8b93cbe 100644 --- a/variables.tf +++ b/variables.tf @@ -49,9 +49,9 @@ variable "database_deletion_protection" { default = true } -variable "create_compute_address_for_mqtt" { +variable "enable_external_workers" { type = bool - description = "Whether to create a compute address for MQTT. It is meant to be used by Service of type LoadBalancer from the GKE cluster to expose the embedded MQTT server to the world. This is only required if you want to run worker outside of the GKE cluster." + description = "Switch this to true if you want to run workers from outside of the GCP infrastructure." default = false }