feat(kubernetes): install argocd to the cluster

.devcontainer/devcontainer.json

@@ -6,7 +6,8 @@
       "installTFsec": true,
       "installTerraformDocs": true
     },
-    "ghcr.io/dhoeric/features/trivy:1": {}
+    "ghcr.io/dhoeric/features/trivy:1": {},
+    "ghcr.io/devcontainers-contrib/features/argo-cd:1": {}
   },
   "customizations": {
     "vscode": {

modules/hetzner/README.md

@@ -72,8 +72,10 @@
 | <a name="output_k3s_cluster_cidr"></a> [k3s\_cluster\_cidr](#output\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster |
 | <a name="output_kube_api_server"></a> [kube\_api\_server](#output\_kube\_api\_server) | Kubernetes API server address |
 | <a name="output_kubeconfig"></a> [kubeconfig](#output\_kubeconfig) | Kubeconfig |
+| <a name="output_location"></a> [location](#output\_location) | Location to use. This is a single datacentre. |
 | <a name="output_network_name"></a> [network\_name](#output\_network\_name) | Name of the network |
 | <a name="output_pools"></a> [pools](#output\_pools) | Servers created |
+| <a name="output_region"></a> [region](#output\_region) | Region to use. This covers multiple datacentres. |
 | <a name="output_ssh_port"></a> [ssh\_port](#output\_ssh\_port) | SSH port for server |
 | <a name="output_ssh_user"></a> [ssh\_user](#output\_ssh\_user) | SSH user for server |
 <!-- END_TF_DOCS -->

modules/hetzner/output.tf

@@ -33,6 +33,11 @@ output "k3s_cluster_cidr" {
   value       = module.k3s.cluster_cidr
 }
 
+output "location" {
+  description = "Location to use. This is a single datacentre."
+  value       = var.location
+}
+
 output "network_name" {
   description = "Name of the network"
   value       = hcloud_network.network.name
@@ -61,6 +66,11 @@ output "pools" {
   )
 }
 
+output "region" {
+  description = "Region to use. This covers multiple datacentres."
+  value       = var.region
+}
+
 output "ssh_port" {
   description = "SSH port for server"
   value       = var.ssh_port

modules/kubernetes/README.md

@@ -16,6 +16,7 @@
 |------|---------|
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.14.1 |
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.31.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.6.2 |
 
 ## Modules
 
@@ -25,21 +26,30 @@ No modules.
 
 | Name | Type |
 |------|------|
+| [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
+| [random_integer.ingress_load_balancer_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_argocd_version"></a> [argocd\_version](#input\_argocd\_version) | Version of ArgoCD to use - defaults to latest | `string` | `null` | no |
+| <a name="input_cluster_issuer"></a> [cluster\_issuer](#input\_cluster\_issuer) | Cluster issuer to use for certificate | `string` | `"letsencrypt-staging"` | no |
+| <a name="input_domain"></a> [domain](#input\_domain) | Domain to use - this may be a top-level or subdomain | `string` | n/a | yes |
 | <a name="input_hcloud_network_name"></a> [hcloud\_network\_name](#input\_hcloud\_network\_name) | Name of the network | `string` | n/a | yes |
 | <a name="input_hcloud_token"></a> [hcloud\_token](#input\_hcloud\_token) | Write token for the Hetzner API | `string` | n/a | yes |
 | <a name="input_hetzner_cloud_config_manager_version"></a> [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no |
 | <a name="input_hetzner_csi_driver_version"></a> [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - defaults to latest | `string` | `null` | no |
+| <a name="input_ingress_nginx_version"></a> [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Version of Ingress Nginx to install - defaults to latest | `string` | `null` | no |
 | <a name="input_k3s_cluster_cidr"></a> [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no |
 | <a name="input_kube_context"></a> [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no |
 | <a name="input_kubeconfig"></a> [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes |
+| <a name="input_load_balancer_location"></a> [load\_balancer\_location](#input\_load\_balancer\_location) | Location to use for the load balancer | `string` | n/a | yes |
+| <a name="input_load_balancer_type"></a> [load\_balancer\_type](#input\_load\_balancer\_type) | Type of load balancer to use | `string` | `"lb11"` | no |
 
 ## Outputs
 

modules/kubernetes/argocd.tf

@@ -0,0 +1,35 @@
+# Copyright 2024 Simon Emms <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "helm_release" "argocd" {
+  chart            = "argo-cd"
+  name             = "argocd"
+  atomic           = true
+  cleanup_on_fail  = true
+  create_namespace = true
+  namespace        = "argocd"
+  repository       = "https://argoproj.github.io/argo-helm"
+  reset_values     = true
+  version          = var.argocd_version
+  wait             = true
+
+  timeout = 10 * 60 # The Redis deployment can take it's sweet time
+
+  values = [
+    templatefile("${path.module}/files/argocd.yaml", {
+      cluster_issuer = var.cluster_issuer
+      domain         = "argocd.${var.domain}"
+    })
+  ]
+}

modules/kubernetes/files/argocd.yaml

@@ -0,0 +1,28 @@
+global:
+  domain: ${domain}
+redis-ha:
+  enabled: true
+repoServer:
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+server:
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations:
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+      nginx.ingress.kubernetes.io/backend-protocol: HTTP
+      kubernetes.io/tls-acme: "true"
+      cert-manager.io/cluster-issuer: ${cluster_issuer}
+    tls: true
+    extraTLS:
+      - hosts:
+          - ${domain}
+        secretName: argocd-tls
+configs:
+  params:
+    server.insecure: true

modules/kubernetes/files/ingress-nginx.yaml

@@ -0,0 +1,14 @@
+# The proxy protocol settings conflict with cert-manager
+# @link https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/issues/354
+controller:
+  kind: DaemonSet
+  config:
+    use-proxy-protocol: false
+  service:
+    annotations:
+      load-balancer.hetzner.cloud/name: "${name}"
+      load-balancer.hetzner.cloud/location: "${location}"
+      load-balancer.hetzner.cloud/type: "${type}"
+      load-balancer.hetzner.cloud/disable-private-ingress: true
+      load-balancer.hetzner.cloud/use-private-ip: true
+      load-balancer.hetzner.cloud/uses-proxyprotocol: false

modules/kubernetes/ingress-nginx.tf

@@ -0,0 +1,47 @@
+# Copyright 2024 Simon Emms <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deploy via Terraform to ensure load balancer is stopped when destroying infra
+
+resource "random_integer" "ingress_load_balancer_id" {
+  min = 1000
+  max = 9999
+}
+
+resource "helm_release" "ingress_nginx" {
+  chart            = "ingress-nginx"
+  name             = "ingress-nginx"
+  atomic           = true
+  cleanup_on_fail  = true
+  create_namespace = true
+  namespace        = "ingress-nginx"
+  repository       = "https://kubernetes.github.io/ingress-nginx"
+  reset_values     = true
+  version          = var.ingress_nginx_version
+  wait             = true
+
+  values = [
+    templatefile("${path.module}/files/ingress-nginx.yaml", {
+      location = var.load_balancer_location
+      name     = "k3s-${random_integer.ingress_load_balancer_id.result}"
+      type     = var.load_balancer_type
+    })
+  ]
+
+  # Depend upon the HCloud CCM to allow the load balancer to be deleted on-destroy
+  depends_on = [
+    helm_release.hcloud_ccm,
+    helm_release.hcloud_csi,
+  ]
+}

modules/kubernetes/variables.tf

@@ -12,6 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+variable "argocd_version" {
+  type        = string
+  description = "Version of ArgoCD to use - defaults to latest"
+  default     = null
+}
+
+variable "cluster_issuer" {
+  type        = string
+  description = "Cluster issuer to use for certificate"
+  default     = "letsencrypt-staging"
+}
+
+variable "domain" {
+  type        = string
+  description = "Domain to use - this may be a top-level or subdomain"
+}
+
 variable "hcloud_network_name" {
   type        = string
   description = "Name of the network"
@@ -52,3 +69,20 @@ variable "kube_context" {
   description = "Kubernetes context to use"
   default     = "default"
 }
+
+variable "ingress_nginx_version" {
+  type        = string
+  description = "Version of Ingress Nginx to install - defaults to latest"
+  default     = null
+}
+
+variable "load_balancer_location" {
+  type        = string
+  description = "Location to use for the load balancer"
+}
+
+variable "load_balancer_type" {
+  type        = string
+  description = "Type of load balancer to use"
+  default     = "lb11"
+}

stacks/dev/hetzner/terragrunt.hcl

@@ -22,7 +22,7 @@ include {
 
 inputs = {
   k3s_manager_pool = {
-    count = 3
+    count = 1
   }
   k3s_worker_pools = [
     {

stacks/dev/kubernetes/terragrunt.hcl

@@ -27,11 +27,14 @@ dependency "hetzner" {
     hcloud_network_name = "some-network-name"
     k3s_cluster_cidr    = "some-cluster-cidr"
     kubeconfig          = "some-kubeconfig"
+    location            = "some-location"
   }
 }
 
 inputs = {
-  hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name
-  k3s_cluster_cidr    = dependency.hetzner.outputs.k3s_cluster_cidr
-  kubeconfig          = dependency.hetzner.outputs.kubeconfig
+  domain                 = "dev.simonemms.com"
+  hcloud_network_name    = dependency.hetzner.outputs.hcloud_network_name
+  k3s_cluster_cidr       = dependency.hetzner.outputs.k3s_cluster_cidr
+  kubeconfig             = dependency.hetzner.outputs.kubeconfig
+  load_balancer_location = dependency.hetzner.outputs.location
 }

stacks/prod/hetzner/terragrunt.hcl

@@ -22,6 +22,12 @@ include {
 
 inputs = {
   k3s_manager_pool = {
-    count = 3
+    count = 1
   }
+  k3s_worker_pools = [
+    {
+      count = 2
+      name  = "pool1"
+    },
+  ]
 }

stacks/prod/kubernetes/terragrunt.hcl

@@ -27,11 +27,15 @@ dependency "hetzner" {
     hcloud_network_name = "some-network-name"
     k3s_cluster_cidr    = "some-cluster-cidr"
     kubeconfig          = "some-kubeconfig"
+    location            = "some-location"
   }
 }
 
 inputs = {
-  hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name
-  k3s_cluster_cidr    = dependency.hetzner.outputs.k3s_cluster_cidr
-  kubeconfig          = dependency.hetzner.outputs.kubeconfig
+  cluster_issuer         = "letsencrypt"
+  domain                 = "prod.simonemms.com"
+  hcloud_network_name    = dependency.hetzner.outputs.hcloud_network_name
+  k3s_cluster_cidr       = dependency.hetzner.outputs.k3s_cluster_cidr
+  kubeconfig             = dependency.hetzner.outputs.kubeconfig
+  load_balancer_location = dependency.hetzner.outputs.location
 }