From 2f09cf84067a4492433127633a1d1df1ae9a5f5a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 10 Jul 2024 15:08:18 +0200 Subject: [PATCH] terraform, gcp: upgrade to google provider, cleanup use of google-beta --- terraform/gcp/cluster.tf | 77 ++++++++++++++++++++++++-------------- terraform/gcp/main.tf | 12 +----- terraform/gcp/variables.tf | 4 +- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/terraform/gcp/cluster.tf b/terraform/gcp/cluster.tf index 75979e1c07..edfacde45b 100644 --- a/terraform/gcp/cluster.tf +++ b/terraform/gcp/cluster.tf @@ -44,11 +44,8 @@ resource "google_project_iam_member" "cluster_sa_roles" { member = "serviceAccount:${google_service_account.cluster_sa.email}" } -# resource ref: https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_container_cluster +# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_container_cluster resource "google_container_cluster" "cluster" { - # Setting cluster autoscaling profile is in google-beta - provider = google-beta - name = "${var.prefix}-cluster" location = var.regional_cluster ? var.region : var.zone node_locations = var.regional_cluster ? [var.zone] : null @@ -176,7 +173,7 @@ resource "google_container_cluster" "cluster" { resource_labels = {} } -# resource ref: https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/container_node_pool +# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool resource "google_container_node_pool" "core" { name = "core-pool" cluster = google_container_cluster.cluster.name @@ -242,7 +239,7 @@ resource "google_container_node_pool" "core" { } } -# resource ref: https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/container_node_pool +# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool resource "google_container_node_pool" "notebook" { for_each = var.notebook_nodes @@ -315,20 +312,33 @@ resource "google_container_node_pool" "notebook" { "k8s.dask.org/node-purpose" = "scheduler", }, each.value.labels) - taint = concat([{ - key = "hub.jupyter.org_dedicated" - value = "user" - effect = "NO_SCHEDULE" - }], - # Add extra taint explicitly if GPU is enabled, so non-GPU pods aren't scheduled here - # Terraform implicitly adds this taint anyway, and tries to recreate the nodepool if we re-apply - each.value.gpu.enabled ? [{ - effect = "NO_SCHEDULE" - key = "nvidia.com/gpu" - value = "present" - }] : [], - each.value.taints - ) + dynamic "taint" { + for_each = concat( + [ + { + key = "hub.jupyter.org_dedicated" + value = "user" + effect = "NO_SCHEDULE" + } + ], + # Add extra taint explicitly if GPU is enabled, so non-GPU pods aren't + # scheduled here Terraform implicitly adds this taint anyway, and tries + # to recreate the nodepool if we re-apply + each.value.gpu.enabled ? [{ + effect = "NO_SCHEDULE" + key = "nvidia.com/gpu" + value = "present" + }] : [], + each.value.taints + ) + + content { + key = taint.value["key"] + value = taint.value["value"] + effect = taint.value["effect"] + } + } + machine_type = each.value.machine_type # Our service account gets all OAuth scopes so it can access @@ -349,7 +359,7 @@ resource "google_container_node_pool" "notebook" { } } -# resource ref: https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/container_node_pool +# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool resource "google_container_node_pool" "dask_worker" { name = "dask-${each.key}" cluster = google_container_cluster.cluster.name @@ -400,13 +410,24 @@ resource "google_container_node_pool" "dask_worker" { "k8s.dask.org/node-purpose" = "worker", }, each.value.labels) - taint = concat([{ - key = "k8s.dask.org_dedicated" - value = "worker" - effect = "NO_SCHEDULE" - }], - each.value.taints - ) + dynamic "taint" { + for_each = concat( + [ + { + key = "k8s.dask.org_dedicated" + value = "worker" + effect = "NO_SCHEDULE" + } + ], + each.value.taints + ) + + content { + key = taint.value["key"] + value = taint.value["value"] + effect = taint.value["effect"] + } + } machine_type = each.value.machine_type diff --git a/terraform/gcp/main.tf b/terraform/gcp/main.tf index 0bc8d99722..8eda10ced3 100644 --- a/terraform/gcp/main.tf +++ b/terraform/gcp/main.tf @@ -5,18 +5,8 @@ terraform { required_providers { google = { # ref: https://registry.terraform.io/providers/hashicorp/google/latest - # - # FIXME: v5 is out but we've not managed to migrate the config yet, we run - # into something about taints. See the upgrade guide at: - # https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/version_5_upgrade. - # source = "google" - version = "~> 4.85" - } - google-beta = { - # ref: https://registry.terraform.io/providers/hashicorp/google-beta/latest - source = "google-beta" - version = "~> 4.85" + version = "~> 5.36" } kubernetes = { # ref: https://registry.terraform.io/providers/hashicorp/kubernetes/latest diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 3bc4c99ae8..9f3ebb3225 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -91,8 +91,8 @@ variable "k8s_versions" { description = <<-EOT Configuration of the k8s cluster's version and node pools' versions. To specify these - - min_master_nodes is passthrough configuration of google_container_cluster's min_master_version, documented in https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/container_cluster#min_master_version - - [core|notebook|dask]_nodes_version is passthrough configuration of container_node_pool's version, documented in https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/container_node_pool#version + - min_master_nodes is passthrough configuration of google_container_cluster's min_master_version, documented in https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#min_master_version + - [core|notebook|dask]_nodes_version is passthrough configuration of container_node_pool's version, documented in https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#version EOT }