From ed7911fc858ec19df97a17761a1ca18d8fc58f67 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Thu, 24 Oct 2024 16:38:43 +0000 Subject: [PATCH] feat(kubernetes): add ingress to kubernetes terraform Normally, this would be in the ArgoCD config, but as this creates a load balancer I want to ensure it gets cleared up when destroying the cluster which is not guaranteed if I run `terraform destroy` --- modules/hetzner/README.md | 2 + modules/hetzner/output.tf | 10 +++++ modules/kubernetes/README.md | 6 +++ modules/kubernetes/files/ingress-nginx.yaml | 14 ++++++ modules/kubernetes/ingress-nginx.tf | 47 +++++++++++++++++++++ modules/kubernetes/variables.tf | 17 ++++++++ stacks/dev/kubernetes/terragrunt.hcl | 10 +++-- stacks/prod/kubernetes/terragrunt.hcl | 8 ++-- 8 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 modules/kubernetes/files/ingress-nginx.yaml create mode 100644 modules/kubernetes/ingress-nginx.tf diff --git a/modules/hetzner/README.md b/modules/hetzner/README.md index 785ca5e..d73943d 100644 --- a/modules/hetzner/README.md +++ b/modules/hetzner/README.md @@ -72,8 +72,10 @@ | [k3s\_cluster\_cidr](#output\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | | [kube\_api\_server](#output\_kube\_api\_server) | Kubernetes API server address | | [kubeconfig](#output\_kubeconfig) | Kubeconfig | +| [location](#output\_location) | Location to use. This is a single datacentre. | | [network\_name](#output\_network\_name) | Name of the network | | [pools](#output\_pools) | Servers created | +| [region](#output\_region) | Region to use. This covers multiple datacentres. | | [ssh\_port](#output\_ssh\_port) | SSH port for server | | [ssh\_user](#output\_ssh\_user) | SSH user for server | diff --git a/modules/hetzner/output.tf b/modules/hetzner/output.tf index 126f678..1931e98 100644 --- a/modules/hetzner/output.tf +++ b/modules/hetzner/output.tf @@ -33,6 +33,11 @@ output "k3s_cluster_cidr" { value = module.k3s.cluster_cidr } +output "location" { + description = "Location to use. This is a single datacentre." + value = var.location +} + output "network_name" { description = "Name of the network" value = hcloud_network.network.name @@ -61,6 +66,11 @@ output "pools" { ) } +output "region" { + description = "Region to use. This covers multiple datacentres." + value = var.region +} + output "ssh_port" { description = "SSH port for server" value = var.ssh_port diff --git a/modules/kubernetes/README.md b/modules/kubernetes/README.md index 329e61e..2d02858 100644 --- a/modules/kubernetes/README.md +++ b/modules/kubernetes/README.md @@ -16,6 +16,7 @@ |------|---------| | [helm](#provider\_helm) | 2.14.1 | | [kubernetes](#provider\_kubernetes) | 2.31.0 | +| [random](#provider\_random) | 3.6.2 | ## Modules @@ -28,7 +29,9 @@ No modules. | [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [random_integer.ingress_load_balancer_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | ## Inputs @@ -40,9 +43,12 @@ No modules. | [hcloud\_token](#input\_hcloud\_token) | Write token for the Hetzner API | `string` | n/a | yes | | [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no | | [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - defaults to latest | `string` | `null` | no | +| [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Version of Ingress Nginx to install - defaults to latest | `string` | `null` | no | | [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no | | [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no | | [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes | +| [load\_balancer\_location](#input\_load\_balancer\_location) | Location to use for the load balancer | `string` | n/a | yes | +| [load\_balancer\_type](#input\_load\_balancer\_type) | Type of load balancer to use | `string` | `"lb11"` | no | ## Outputs diff --git a/modules/kubernetes/files/ingress-nginx.yaml b/modules/kubernetes/files/ingress-nginx.yaml new file mode 100644 index 0000000..25e68dc --- /dev/null +++ b/modules/kubernetes/files/ingress-nginx.yaml @@ -0,0 +1,14 @@ +# The proxy protocol settings conflict with cert-manager +# @link https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/issues/354 +controller: + kind: DaemonSet + config: + use-proxy-protocol: false + service: + annotations: + load-balancer.hetzner.cloud/name: "${name}" + load-balancer.hetzner.cloud/location: "${location}" + load-balancer.hetzner.cloud/type: "${type}" + load-balancer.hetzner.cloud/disable-private-ingress: true + load-balancer.hetzner.cloud/use-private-ip: true + load-balancer.hetzner.cloud/uses-proxyprotocol: false diff --git a/modules/kubernetes/ingress-nginx.tf b/modules/kubernetes/ingress-nginx.tf new file mode 100644 index 0000000..6de3a8e --- /dev/null +++ b/modules/kubernetes/ingress-nginx.tf @@ -0,0 +1,47 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deploy via Terraform to ensure load balancer is stopped when destroying infra + +resource "random_integer" "ingress_load_balancer_id" { + min = 1000 + max = 9999 +} + +resource "helm_release" "ingress_nginx" { + chart = "ingress-nginx" + name = "ingress-nginx" + atomic = true + cleanup_on_fail = true + create_namespace = true + namespace = "ingress-nginx" + repository = "https://kubernetes.github.io/ingress-nginx" + reset_values = true + version = var.ingress_nginx_version + wait = true + + values = [ + templatefile("${path.module}/files/ingress-nginx.yaml", { + location = var.load_balancer_location + name = "k3s-${random_integer.ingress_load_balancer_id.result}" + type = var.load_balancer_type + }) + ] + + # Depend upon the HCloud CCM to allow the load balancer to be deleted on-destroy + depends_on = [ + helm_release.hcloud_ccm, + helm_release.hcloud_csi, + ] +} diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index 753e904..fce7229 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -63,3 +63,20 @@ variable "kube_context" { description = "Kubernetes context to use" default = "default" } + +variable "ingress_nginx_version" { + type = string + description = "Version of Ingress Nginx to install - defaults to latest" + default = null +} + +variable "load_balancer_location" { + type = string + description = "Location to use for the load balancer" +} + +variable "load_balancer_type" { + type = string + description = "Type of load balancer to use" + default = "lb11" +} diff --git a/stacks/dev/kubernetes/terragrunt.hcl b/stacks/dev/kubernetes/terragrunt.hcl index 04829f0..54edecb 100644 --- a/stacks/dev/kubernetes/terragrunt.hcl +++ b/stacks/dev/kubernetes/terragrunt.hcl @@ -27,12 +27,14 @@ dependency "hetzner" { hcloud_network_name = "some-network-name" k3s_cluster_cidr = "some-cluster-cidr" kubeconfig = "some-kubeconfig" + location = "some-location" } } inputs = { - domain = "dev.simonemms.com" - hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name - k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr - kubeconfig = dependency.hetzner.outputs.kubeconfig + domain = "dev.simonemms.com" + hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name + k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr + kubeconfig = dependency.hetzner.outputs.kubeconfig + load_balancer_location = dependency.hetzner.outputs.location } diff --git a/stacks/prod/kubernetes/terragrunt.hcl b/stacks/prod/kubernetes/terragrunt.hcl index 840ade4..97f1044 100644 --- a/stacks/prod/kubernetes/terragrunt.hcl +++ b/stacks/prod/kubernetes/terragrunt.hcl @@ -27,11 +27,13 @@ dependency "hetzner" { hcloud_network_name = "some-network-name" k3s_cluster_cidr = "some-cluster-cidr" kubeconfig = "some-kubeconfig" + location = "some-location" } } inputs = { - hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name - k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr - kubeconfig = dependency.hetzner.outputs.kubeconfig + hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name + k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr + kubeconfig = dependency.hetzner.outputs.kubeconfig + load_balancer_location = dependency.hetzner.outputs.location }