From 94a943f488cf708593f711495cd726053f037495 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Tue, 22 Oct 2024 16:52:06 +0000 Subject: [PATCH 1/4] feat(kubernetes): install argocd to the cluster --- .devcontainer/devcontainer.json | 3 ++- modules/kubernetes/README.md | 3 +++ modules/kubernetes/argocd.tf | 34 ++++++++++++++++++++++++++++ modules/kubernetes/files/argocd.yaml | 28 +++++++++++++++++++++++ modules/kubernetes/variables.tf | 11 +++++++++ stacks/dev/kubernetes/terragrunt.hcl | 1 + 6 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 modules/kubernetes/argocd.tf create mode 100644 modules/kubernetes/files/argocd.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 77eb5ff..29e3776 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,8 @@ "installTFsec": true, "installTerraformDocs": true }, - "ghcr.io/dhoeric/features/trivy:1": {} + "ghcr.io/dhoeric/features/trivy:1": {}, + "ghcr.io/devcontainers-contrib/features/argo-cd:1": {} }, "customizations": { "vscode": { diff --git a/modules/kubernetes/README.md b/modules/kubernetes/README.md index 3d20253..329e61e 100644 --- a/modules/kubernetes/README.md +++ b/modules/kubernetes/README.md @@ -25,6 +25,7 @@ No modules. | Name | Type | |------|------| +| [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | @@ -33,6 +34,8 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [argocd\_version](#input\_argocd\_version) | Version of ArgoCD to use - defaults to latest | `string` | `null` | no | +| [domain](#input\_domain) | Domain to use - this may be a top-level or subdomain | `string` | n/a | yes | | [hcloud\_network\_name](#input\_hcloud\_network\_name) | Name of the network | `string` | n/a | yes | | [hcloud\_token](#input\_hcloud\_token) | Write token for the Hetzner API | `string` | n/a | yes | | [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no | diff --git a/modules/kubernetes/argocd.tf b/modules/kubernetes/argocd.tf new file mode 100644 index 0000000..0988b18 --- /dev/null +++ b/modules/kubernetes/argocd.tf @@ -0,0 +1,34 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "helm_release" "argocd" { + chart = "argo-cd" + name = "argocd" + atomic = true + cleanup_on_fail = true + create_namespace = true + namespace = "argocd" + repository = "https://argoproj.github.io/argo-helm" + reset_values = true + version = var.argocd_version + wait = true + + timeout = 10 * 60 # The Redis deployment can take it's sweet time + + values = [ + templatefile("${path.module}/files/argocd.yaml", { + domain = "argocd.${var.domain}" + }) + ] +} diff --git a/modules/kubernetes/files/argocd.yaml b/modules/kubernetes/files/argocd.yaml new file mode 100644 index 0000000..29f4242 --- /dev/null +++ b/modules/kubernetes/files/argocd.yaml @@ -0,0 +1,28 @@ +global: + domain: ${domain} +redis-ha: + enabled: true +repoServer: + autoscaling: + enabled: true + minReplicas: 2 +server: + autoscaling: + enabled: true + minReplicas: 2 + ingress: + enabled: true + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: HTTP + kubernetes.io/tls-acme: "true" + cert-manager.io/cluster-issuer: letsencrypt + tls: true + extraTLS: + - hosts: + - ${domain} + secretName: argocd-tls +configs: + params: + server.insecure: true diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index 4d1f0b2..753e904 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -12,6 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +variable "argocd_version" { + type = string + description = "Version of ArgoCD to use - defaults to latest" + default = null +} + +variable "domain" { + type = string + description = "Domain to use - this may be a top-level or subdomain" +} + variable "hcloud_network_name" { type = string description = "Name of the network" diff --git a/stacks/dev/kubernetes/terragrunt.hcl b/stacks/dev/kubernetes/terragrunt.hcl index 840ade4..04829f0 100644 --- a/stacks/dev/kubernetes/terragrunt.hcl +++ b/stacks/dev/kubernetes/terragrunt.hcl @@ -31,6 +31,7 @@ dependency "hetzner" { } inputs = { + domain = "dev.simonemms.com" hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr kubeconfig = dependency.hetzner.outputs.kubeconfig From 1fd6a34fac30a60a1b2696f663f0818f0c388517 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Thu, 24 Oct 2024 16:38:43 +0000 Subject: [PATCH 2/4] feat(kubernetes): add ingress to kubernetes terraform Normally, this would be in the ArgoCD config, but as this creates a load balancer I want to ensure it gets cleared up when destroying the cluster which is not guaranteed if I run `terraform destroy` --- modules/hetzner/README.md | 2 + modules/hetzner/output.tf | 10 +++++ modules/kubernetes/README.md | 6 +++ modules/kubernetes/files/ingress-nginx.yaml | 14 ++++++ modules/kubernetes/ingress-nginx.tf | 47 +++++++++++++++++++++ modules/kubernetes/variables.tf | 17 ++++++++ stacks/dev/kubernetes/terragrunt.hcl | 10 +++-- stacks/prod/kubernetes/terragrunt.hcl | 9 ++-- 8 files changed, 108 insertions(+), 7 deletions(-) create mode 100644 modules/kubernetes/files/ingress-nginx.yaml create mode 100644 modules/kubernetes/ingress-nginx.tf diff --git a/modules/hetzner/README.md b/modules/hetzner/README.md index 785ca5e..d73943d 100644 --- a/modules/hetzner/README.md +++ b/modules/hetzner/README.md @@ -72,8 +72,10 @@ | [k3s\_cluster\_cidr](#output\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | | [kube\_api\_server](#output\_kube\_api\_server) | Kubernetes API server address | | [kubeconfig](#output\_kubeconfig) | Kubeconfig | +| [location](#output\_location) | Location to use. This is a single datacentre. | | [network\_name](#output\_network\_name) | Name of the network | | [pools](#output\_pools) | Servers created | +| [region](#output\_region) | Region to use. This covers multiple datacentres. | | [ssh\_port](#output\_ssh\_port) | SSH port for server | | [ssh\_user](#output\_ssh\_user) | SSH user for server | diff --git a/modules/hetzner/output.tf b/modules/hetzner/output.tf index 126f678..1931e98 100644 --- a/modules/hetzner/output.tf +++ b/modules/hetzner/output.tf @@ -33,6 +33,11 @@ output "k3s_cluster_cidr" { value = module.k3s.cluster_cidr } +output "location" { + description = "Location to use. This is a single datacentre." + value = var.location +} + output "network_name" { description = "Name of the network" value = hcloud_network.network.name @@ -61,6 +66,11 @@ output "pools" { ) } +output "region" { + description = "Region to use. This covers multiple datacentres." + value = var.region +} + output "ssh_port" { description = "SSH port for server" value = var.ssh_port diff --git a/modules/kubernetes/README.md b/modules/kubernetes/README.md index 329e61e..2d02858 100644 --- a/modules/kubernetes/README.md +++ b/modules/kubernetes/README.md @@ -16,6 +16,7 @@ |------|---------| | [helm](#provider\_helm) | 2.14.1 | | [kubernetes](#provider\_kubernetes) | 2.31.0 | +| [random](#provider\_random) | 3.6.2 | ## Modules @@ -28,7 +29,9 @@ No modules. | [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [random_integer.ingress_load_balancer_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource | ## Inputs @@ -40,9 +43,12 @@ No modules. | [hcloud\_token](#input\_hcloud\_token) | Write token for the Hetzner API | `string` | n/a | yes | | [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no | | [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - defaults to latest | `string` | `null` | no | +| [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Version of Ingress Nginx to install - defaults to latest | `string` | `null` | no | | [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no | | [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no | | [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes | +| [load\_balancer\_location](#input\_load\_balancer\_location) | Location to use for the load balancer | `string` | n/a | yes | +| [load\_balancer\_type](#input\_load\_balancer\_type) | Type of load balancer to use | `string` | `"lb11"` | no | ## Outputs diff --git a/modules/kubernetes/files/ingress-nginx.yaml b/modules/kubernetes/files/ingress-nginx.yaml new file mode 100644 index 0000000..25e68dc --- /dev/null +++ b/modules/kubernetes/files/ingress-nginx.yaml @@ -0,0 +1,14 @@ +# The proxy protocol settings conflict with cert-manager +# @link https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/issues/354 +controller: + kind: DaemonSet + config: + use-proxy-protocol: false + service: + annotations: + load-balancer.hetzner.cloud/name: "${name}" + load-balancer.hetzner.cloud/location: "${location}" + load-balancer.hetzner.cloud/type: "${type}" + load-balancer.hetzner.cloud/disable-private-ingress: true + load-balancer.hetzner.cloud/use-private-ip: true + load-balancer.hetzner.cloud/uses-proxyprotocol: false diff --git a/modules/kubernetes/ingress-nginx.tf b/modules/kubernetes/ingress-nginx.tf new file mode 100644 index 0000000..6de3a8e --- /dev/null +++ b/modules/kubernetes/ingress-nginx.tf @@ -0,0 +1,47 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deploy via Terraform to ensure load balancer is stopped when destroying infra + +resource "random_integer" "ingress_load_balancer_id" { + min = 1000 + max = 9999 +} + +resource "helm_release" "ingress_nginx" { + chart = "ingress-nginx" + name = "ingress-nginx" + atomic = true + cleanup_on_fail = true + create_namespace = true + namespace = "ingress-nginx" + repository = "https://kubernetes.github.io/ingress-nginx" + reset_values = true + version = var.ingress_nginx_version + wait = true + + values = [ + templatefile("${path.module}/files/ingress-nginx.yaml", { + location = var.load_balancer_location + name = "k3s-${random_integer.ingress_load_balancer_id.result}" + type = var.load_balancer_type + }) + ] + + # Depend upon the HCloud CCM to allow the load balancer to be deleted on-destroy + depends_on = [ + helm_release.hcloud_ccm, + helm_release.hcloud_csi, + ] +} diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index 753e904..fce7229 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -63,3 +63,20 @@ variable "kube_context" { description = "Kubernetes context to use" default = "default" } + +variable "ingress_nginx_version" { + type = string + description = "Version of Ingress Nginx to install - defaults to latest" + default = null +} + +variable "load_balancer_location" { + type = string + description = "Location to use for the load balancer" +} + +variable "load_balancer_type" { + type = string + description = "Type of load balancer to use" + default = "lb11" +} diff --git a/stacks/dev/kubernetes/terragrunt.hcl b/stacks/dev/kubernetes/terragrunt.hcl index 04829f0..54edecb 100644 --- a/stacks/dev/kubernetes/terragrunt.hcl +++ b/stacks/dev/kubernetes/terragrunt.hcl @@ -27,12 +27,14 @@ dependency "hetzner" { hcloud_network_name = "some-network-name" k3s_cluster_cidr = "some-cluster-cidr" kubeconfig = "some-kubeconfig" + location = "some-location" } } inputs = { - domain = "dev.simonemms.com" - hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name - k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr - kubeconfig = dependency.hetzner.outputs.kubeconfig + domain = "dev.simonemms.com" + hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name + k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr + kubeconfig = dependency.hetzner.outputs.kubeconfig + load_balancer_location = dependency.hetzner.outputs.location } diff --git a/stacks/prod/kubernetes/terragrunt.hcl b/stacks/prod/kubernetes/terragrunt.hcl index 840ade4..a92f424 100644 --- a/stacks/prod/kubernetes/terragrunt.hcl +++ b/stacks/prod/kubernetes/terragrunt.hcl @@ -27,11 +27,14 @@ dependency "hetzner" { hcloud_network_name = "some-network-name" k3s_cluster_cidr = "some-cluster-cidr" kubeconfig = "some-kubeconfig" + location = "some-location" } } inputs = { - hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name - k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr - kubeconfig = dependency.hetzner.outputs.kubeconfig + domain = "prod.simonemms.com" + hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name + k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr + kubeconfig = dependency.hetzner.outputs.kubeconfig + load_balancer_location = dependency.hetzner.outputs.location } From 0186791280cde84a335d5e2b631582c94f79c729 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Fri, 1 Nov 2024 16:30:08 +0000 Subject: [PATCH 3/4] feat: provide the cert-issuer as a variable --- modules/kubernetes/README.md | 1 + modules/kubernetes/argocd.tf | 3 ++- modules/kubernetes/files/argocd.yaml | 2 +- modules/kubernetes/variables.tf | 6 ++++++ stacks/prod/kubernetes/terragrunt.hcl | 1 + 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/kubernetes/README.md b/modules/kubernetes/README.md index 2d02858..7ae8508 100644 --- a/modules/kubernetes/README.md +++ b/modules/kubernetes/README.md @@ -38,6 +38,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [argocd\_version](#input\_argocd\_version) | Version of ArgoCD to use - defaults to latest | `string` | `null` | no | +| [cluster\_issuer](#input\_cluster\_issuer) | Cluster issuer to use for certificate | `string` | `"letsencrypt-staging"` | no | | [domain](#input\_domain) | Domain to use - this may be a top-level or subdomain | `string` | n/a | yes | | [hcloud\_network\_name](#input\_hcloud\_network\_name) | Name of the network | `string` | n/a | yes | | [hcloud\_token](#input\_hcloud\_token) | Write token for the Hetzner API | `string` | n/a | yes | diff --git a/modules/kubernetes/argocd.tf b/modules/kubernetes/argocd.tf index 0988b18..6ccadf8 100644 --- a/modules/kubernetes/argocd.tf +++ b/modules/kubernetes/argocd.tf @@ -28,7 +28,8 @@ resource "helm_release" "argocd" { values = [ templatefile("${path.module}/files/argocd.yaml", { - domain = "argocd.${var.domain}" + cluster_issuer = var.cluster_issuer + domain = "argocd.${var.domain}" }) ] } diff --git a/modules/kubernetes/files/argocd.yaml b/modules/kubernetes/files/argocd.yaml index 29f4242..49274e7 100644 --- a/modules/kubernetes/files/argocd.yaml +++ b/modules/kubernetes/files/argocd.yaml @@ -17,7 +17,7 @@ server: nginx.ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: HTTP kubernetes.io/tls-acme: "true" - cert-manager.io/cluster-issuer: letsencrypt + cert-manager.io/cluster-issuer: ${cluster_issuer} tls: true extraTLS: - hosts: diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index fce7229..2d97652 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -18,6 +18,12 @@ variable "argocd_version" { default = null } +variable "cluster_issuer" { + type = string + description = "Cluster issuer to use for certificate" + default = "letsencrypt-staging" +} + variable "domain" { type = string description = "Domain to use - this may be a top-level or subdomain" diff --git a/stacks/prod/kubernetes/terragrunt.hcl b/stacks/prod/kubernetes/terragrunt.hcl index a92f424..b5bf1dd 100644 --- a/stacks/prod/kubernetes/terragrunt.hcl +++ b/stacks/prod/kubernetes/terragrunt.hcl @@ -32,6 +32,7 @@ dependency "hetzner" { } inputs = { + cluster_issuer = "letsencrypt" domain = "prod.simonemms.com" hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr From b51cc24b99747f187106d3ed94d5da528f423d96 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Fri, 1 Nov 2024 16:32:56 +0000 Subject: [PATCH 4/4] feat: resize the node pools Use a single manager node as that's fine for our requirements --- stacks/dev/hetzner/terragrunt.hcl | 2 +- stacks/prod/hetzner/terragrunt.hcl | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/stacks/dev/hetzner/terragrunt.hcl b/stacks/dev/hetzner/terragrunt.hcl index 31e3285..da0ca37 100644 --- a/stacks/dev/hetzner/terragrunt.hcl +++ b/stacks/dev/hetzner/terragrunt.hcl @@ -22,7 +22,7 @@ include { inputs = { k3s_manager_pool = { - count = 3 + count = 1 } k3s_worker_pools = [ { diff --git a/stacks/prod/hetzner/terragrunt.hcl b/stacks/prod/hetzner/terragrunt.hcl index e6338f7..ca03fdd 100644 --- a/stacks/prod/hetzner/terragrunt.hcl +++ b/stacks/prod/hetzner/terragrunt.hcl @@ -22,6 +22,12 @@ include { inputs = { k3s_manager_pool = { - count = 3 + count = 1 } + k3s_worker_pools = [ + { + count = 2 + name = "pool1" + }, + ] }