feat(kubernetes): add ingress to kubernetes terraform

Normally, this would be in the ArgoCD config, but as this
creates a load balancer I want to ensure it gets cleared
up when destroying the cluster which is not guaranteed if
I run `terraform destroy`

modules/hetzner/README.md

@@ -72,8 +72,10 @@
 | <a name="output_k3s_cluster_cidr"></a> [k3s\_cluster\_cidr](#output\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster |
 | <a name="output_kube_api_server"></a> [kube\_api\_server](#output\_kube\_api\_server) | Kubernetes API server address |
 | <a name="output_kubeconfig"></a> [kubeconfig](#output\_kubeconfig) | Kubeconfig |
+| <a name="output_location"></a> [location](#output\_location) | Location to use. This is a single datacentre. |
 | <a name="output_network_name"></a> [network\_name](#output\_network\_name) | Name of the network |
 | <a name="output_pools"></a> [pools](#output\_pools) | Servers created |
+| <a name="output_region"></a> [region](#output\_region) | Region to use. This covers multiple datacentres. |
 | <a name="output_ssh_port"></a> [ssh\_port](#output\_ssh\_port) | SSH port for server |
 | <a name="output_ssh_user"></a> [ssh\_user](#output\_ssh\_user) | SSH user for server |
 <!-- END_TF_DOCS -->

modules/hetzner/output.tf

@@ -33,6 +33,11 @@ output "k3s_cluster_cidr" {
   value       = module.k3s.cluster_cidr
 }
 
+output "location" {
+  description = "Location to use. This is a single datacentre."
+  value       = var.location
+}
+
 output "network_name" {
   description = "Name of the network"
   value       = hcloud_network.network.name
@@ -61,6 +66,11 @@ output "pools" {
   )
 }
 
+output "region" {
+  description = "Region to use. This covers multiple datacentres."
+  value       = var.region
+}
+
 output "ssh_port" {
   description = "SSH port for server"
   value       = var.ssh_port

modules/kubernetes/README.md

@@ -16,6 +16,7 @@
 |------|---------|
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.14.1 |
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.31.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.6.2 |
 
 ## Modules
 
@@ -28,7 +29,9 @@ No modules.
 | [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
+| [random_integer.ingress_load_balancer_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
 
 ## Inputs
 
@@ -40,9 +43,12 @@ No modules.
 | <a name="input_hcloud_token"></a> [hcloud\_token](#input\_hcloud\_token) | Write token for the Hetzner API | `string` | n/a | yes |
 | <a name="input_hetzner_cloud_config_manager_version"></a> [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no |
 | <a name="input_hetzner_csi_driver_version"></a> [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - defaults to latest | `string` | `null` | no |
+| <a name="input_ingress_nginx_version"></a> [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Version of Ingress Nginx to install - defaults to latest | `string` | `null` | no |
 | <a name="input_k3s_cluster_cidr"></a> [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no |
 | <a name="input_kube_context"></a> [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no |
 | <a name="input_kubeconfig"></a> [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes |
+| <a name="input_load_balancer_location"></a> [load\_balancer\_location](#input\_load\_balancer\_location) | Location to use for the load balancer | `string` | n/a | yes |
+| <a name="input_load_balancer_type"></a> [load\_balancer\_type](#input\_load\_balancer\_type) | Type of load balancer to use | `string` | `"lb11"` | no |
 
 ## Outputs
 

modules/kubernetes/files/ingress-nginx.yaml

@@ -0,0 +1,14 @@
+# The proxy protocol settings conflict with cert-manager
+# @link https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/issues/354
+controller:
+  kind: DaemonSet
+  config:
+    use-proxy-protocol: false
+  service:
+    annotations:
+      load-balancer.hetzner.cloud/name: "${name}"
+      load-balancer.hetzner.cloud/location: "${location}"
+      load-balancer.hetzner.cloud/type: "${type}"
+      load-balancer.hetzner.cloud/disable-private-ingress: true
+      load-balancer.hetzner.cloud/use-private-ip: true
+      load-balancer.hetzner.cloud/uses-proxyprotocol: false

modules/kubernetes/ingress-nginx.tf

@@ -0,0 +1,47 @@
+# Copyright 2024 Simon Emms <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deploy via Terraform to ensure load balancer is stopped when destroying infra
+
+resource "random_integer" "ingress_load_balancer_id" {
+  min = 1000
+  max = 9999
+}
+
+resource "helm_release" "ingress_nginx" {
+  chart            = "ingress-nginx"
+  name             = "ingress-nginx"
+  atomic           = true
+  cleanup_on_fail  = true
+  create_namespace = true
+  namespace        = "ingress-nginx"
+  repository       = "https://kubernetes.github.io/ingress-nginx"
+  reset_values     = true
+  version          = var.ingress_nginx_version
+  wait             = true
+
+  values = [
+    templatefile("${path.module}/files/ingress-nginx.yaml", {
+      location = var.load_balancer_location
+      name     = "k3s-${random_integer.ingress_load_balancer_id.result}"
+      type     = var.load_balancer_type
+    })
+  ]
+
+  # Depend upon the HCloud CCM to allow the load balancer to be deleted on-destroy
+  depends_on = [
+    helm_release.hcloud_ccm,
+    helm_release.hcloud_csi,
+  ]
+}

modules/kubernetes/variables.tf

@@ -63,3 +63,20 @@ variable "kube_context" {
   description = "Kubernetes context to use"
   default     = "default"
 }
+
+variable "ingress_nginx_version" {
+  type        = string
+  description = "Version of Ingress Nginx to install - defaults to latest"
+  default     = null
+}
+
+variable "load_balancer_location" {
+  type        = string
+  description = "Location to use for the load balancer"
+}
+
+variable "load_balancer_type" {
+  type        = string
+  description = "Type of load balancer to use"
+  default     = "lb11"
+}

stacks/dev/kubernetes/terragrunt.hcl

@@ -27,12 +27,14 @@ dependency "hetzner" {
     hcloud_network_name = "some-network-name"
     k3s_cluster_cidr    = "some-cluster-cidr"
     kubeconfig          = "some-kubeconfig"
+    location            = "some-location"
   }
 }
 
 inputs = {
-  domain              = "dev.simonemms.com"
-  hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name
-  k3s_cluster_cidr    = dependency.hetzner.outputs.k3s_cluster_cidr
-  kubeconfig          = dependency.hetzner.outputs.kubeconfig
+  domain                 = "dev.simonemms.com"
+  hcloud_network_name    = dependency.hetzner.outputs.hcloud_network_name
+  k3s_cluster_cidr       = dependency.hetzner.outputs.k3s_cluster_cidr
+  kubeconfig             = dependency.hetzner.outputs.kubeconfig
+  load_balancer_location = dependency.hetzner.outputs.location
 }

stacks/prod/kubernetes/terragrunt.hcl

@@ -27,11 +27,13 @@ dependency "hetzner" {
     hcloud_network_name = "some-network-name"
     k3s_cluster_cidr    = "some-cluster-cidr"
     kubeconfig          = "some-kubeconfig"
+    location            = "some-location"
   }
 }
 
 inputs = {
-  hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name
-  k3s_cluster_cidr    = dependency.hetzner.outputs.k3s_cluster_cidr
-  kubeconfig          = dependency.hetzner.outputs.kubeconfig
+  hcloud_network_name    = dependency.hetzner.outputs.hcloud_network_name
+  k3s_cluster_cidr       = dependency.hetzner.outputs.k3s_cluster_cidr
+  kubeconfig             = dependency.hetzner.outputs.kubeconfig
+  load_balancer_location = dependency.hetzner.outputs.location
 }