Add on-prem Raspberry Pi nodes to cluster

modules/hetzner/README.md

@@ -41,7 +41,10 @@
 | [hcloud_server.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource |
 | [hcloud_ssh_key.server](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/ssh_key) | resource |
 | [local_sensitive_file.kubeconfig](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/sensitive_file) | resource |
+| [ssh_resource.configure_rpi_for_k3s](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 | [ssh_resource.manager_ready](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
+| [ssh_resource.rpi_ready](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
+| [ssh_resource.uninstall_k3s](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 | [ssh_resource.workers_ready](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 
 ## Inputs
@@ -50,6 +53,7 @@
 |------|-------------|------|---------|:--------:|
 | <a name="input_firewall_allow_api_access"></a> [firewall\_allow\_api\_access](#input\_firewall\_allow\_api\_access) | CIDR range to allow access to the Kubernetes API | `list(string)` | <pre>[<br>  "0.0.0.0/0",<br>  "::/0"<br>]</pre> | no |
 | <a name="input_firewall_allow_ssh_access"></a> [firewall\_allow\_ssh\_access](#input\_firewall\_allow\_ssh\_access) | CIDR range to allow access to the servers via SSH | `list(string)` | <pre>[<br>  "0.0.0.0/0",<br>  "::/0"<br>]</pre> | no |
+| <a name="input_k3s_existing_worker_pools"></a> [k3s\_existing\_worker\_pools](#input\_k3s\_existing\_worker\_pools) | Additional workers on already existing nodes | <pre>map(list(object({<br>    name = string<br>    host = string<br>    port = optional(number, 22)<br>    user = optional(string)<br>  })))</pre> | `{}` | no |
 | <a name="input_k3s_manager_load_balancer_algorithm"></a> [k3s\_manager\_load\_balancer\_algorithm](#input\_k3s\_manager\_load\_balancer\_algorithm) | Algorithm to use for the k3s manager load balancer | `string` | `"round_robin"` | no |
 | <a name="input_k3s_manager_load_balancer_type"></a> [k3s\_manager\_load\_balancer\_type](#input\_k3s\_manager\_load\_balancer\_type) | Load balancer type for the k3s manager nodes | `string` | `"lb11"` | no |
 | <a name="input_k3s_manager_pool"></a> [k3s\_manager\_pool](#input\_k3s\_manager\_pool) | Manager pool configuration | <pre>object({<br>    name        = optional(string, "manager")<br>    server_type = optional(string, "cx22")<br>    count       = optional(number, 1)<br>    image       = optional(string, "ubuntu-24.04")<br>  })</pre> | `{}` | no |

modules/hetzner/k3s.tf

@@ -22,6 +22,12 @@ module "k3s" {
       name              = i.name
       node-external-ip  = i.ipv4_address
       node-ip           = tolist(i.network)[0].ip
+      labels = [
+        {
+          key   = "provider"
+          value = "hetzner"
+        }
+      ]
 
       connection = {
         host        = i.ipv4_address
@@ -32,20 +38,38 @@ module "k3s" {
     }
   ]
 
-  workers = {
-    for i, p in local.k3s_worker_pools : p.pool => {
-      name             = hcloud_server.workers[i].name
-      node-external-ip = hcloud_server.workers[i].ipv4_address
-      node-ip          = tolist(hcloud_server.workers[i].network)[0].ip
+  workers = merge(
+    {
+      for i, p in local.k3s_worker_pools : p.pool => {
+        name             = hcloud_server.workers[i].name
+        node-external-ip = hcloud_server.workers[i].ipv4_address
+        node-ip          = tolist(hcloud_server.workers[i].network)[0].ip
+        labels           = lookup(p, "labels", [])
 
-      connection = {
-        host        = hcloud_server.workers[i].ipv4_address
-        port        = var.ssh_port
-        private_key = var.ssh_key
-        user        = local.ssh_user
-      }
-    }...
-  }
+        connection = {
+          host        = hcloud_server.workers[i].ipv4_address
+          port        = var.ssh_port
+          private_key = var.ssh_key
+          user        = local.ssh_user
+        }
+      }...
+    },
+    {
+      for p in local.k3s_additional_pools : p.pool => {
+        name             = p.name
+        node-external-ip = p.host
+        node-ip          = p.host
+        labels           = lookup(p, "labels", [])
+
+        connection = {
+          host        = p.host
+          port        = p.port
+          private_key = var.ssh_key
+          user        = p.user
+        }
+      }...
+    }
+  )
 
   disable_addons = [
     "local-storage",
@@ -59,7 +83,8 @@ module "k3s" {
 
   depends_on = [
     ssh_resource.manager_ready,
-    ssh_resource.workers_ready
+    ssh_resource.workers_ready,
+    ssh_resource.rpi_ready
   ]
 }
 

modules/hetzner/locals.tf

@@ -31,10 +31,35 @@ locals {
           location = w.location != null ? w.location : var.location
           name     = "${w.name}-${n}"
           pool     = w.name
+          labels = [
+            {
+              key   = "provider"
+              value = "hetzner"
+            }
+          ]
         }
       )
     ]
   ])
+  k3s_additional_pools = flatten([
+    for poolName, nodes in var.k3s_existing_worker_pools : [
+      for i, n in nodes : {
+        name = lookup(n, "name", "${poolName}-${i}")
+        pool = poolName
+        labels = [
+          {
+            key   = "provider"
+            value = "manual"
+          }
+        ]
+
+        # Connection details
+        host = n.host
+        port = n.port
+        user = n.user == null ? local.ssh_user : n.user
+      }
+    ]
+  ])
   kubernetes_api_port = 6443
   labels = {
     format(local.label_namespace, "project")     = var.name

modules/hetzner/server.tf

@@ -160,3 +160,77 @@ resource "ssh_resource" "workers_ready" {
 
   depends_on = [hcloud_server.workers]
 }
+
+
+resource "ssh_resource" "configure_rpi_for_k3s" {
+  for_each = { for i in local.k3s_additional_pools : i.host => i }
+
+  host        = each.value.host
+  user        = each.value.user
+  private_key = var.ssh_key
+  port        = each.value.port
+
+  timeout     = "5m"
+  retry_delay = "5s"
+
+  commands = concat(
+    [
+      for i in [
+        "cgroup_enable=cpuset",
+        "cgroup_memory=1",
+        "cgroup_enable=memory"
+      ] : "grep -qF '${i}' /boot/firmware/cmdline.txt || sudo sed -i '1 s/$/ ${i}/' /boot/firmware/cmdline.txt"
+    ],
+    [
+      "sudo reboot"
+    ]
+  )
+}
+
+# This exists to check that the node has restarted
+resource "ssh_resource" "rpi_ready" {
+  for_each = { for i in local.k3s_additional_pools : i.host => i }
+
+  host        = each.value.host
+  user        = each.value.user
+  private_key = var.ssh_key
+  port        = each.value.port
+
+  timeout     = "5m"
+  retry_delay = "5s"
+
+  commands = [
+    "cat /proc/cgroups"
+  ]
+
+  triggers = {
+    always = timestamp()
+  }
+
+  depends_on = [ssh_resource.configure_rpi_for_k3s]
+}
+
+resource "ssh_resource" "uninstall_k3s" {
+  for_each = { for i in local.k3s_additional_pools : i.host => i }
+
+  host        = each.value.host
+  user        = each.value.user
+  private_key = var.ssh_key
+  port        = each.value.port
+
+  timeout     = "5m"
+  retry_delay = "5s"
+
+  commands = [
+    "k3s-agent-uninstall.sh",
+    "sudo rm -Rf /etc/rancher"
+  ]
+
+  triggers = {
+    host = each.key
+  }
+
+  depends_on = [module.k3s]
+
+  when = "destroy"
+}

modules/hetzner/variables.tf

@@ -71,6 +71,17 @@ variable "k3s_worker_pools" {
   default     = []
 }
 
+variable "k3s_existing_worker_pools" {
+  type = map(list(object({
+    name = string
+    host = string
+    port = optional(number, 22)
+    user = optional(string)
+  })))
+  description = "Additional workers on already existing nodes"
+  default     = {}
+}
+
 variable "location" {
   type        = string
   description = "Location to use. This is a single datacentre."

modules/kubernetes/hetzner.tf

@@ -50,6 +50,11 @@ resource "helm_release" "hcloud_ccm" {
     value = sha512(yamlencode(kubernetes_secret_v1.hcloud.data))
   }
 
+  set {
+    name  = "nodeSelector.provider"
+    value = "hetzner"
+  }
+
   depends_on = [kubernetes_secret_v1.hcloud]
 }
 
@@ -69,6 +74,16 @@ resource "helm_release" "hcloud_csi" {
     value = sha512(yamlencode(kubernetes_secret_v1.hcloud.data))
   }
 
+  set {
+    name  = "controller.nodeSelector.provider"
+    value = "hetzner"
+  }
+
+  set {
+    name  = "node.nodeSelector.provider"
+    value = "hetzner"
+  }
+
   # Allow running on control plane nodes
   dynamic "set" {
     for_each = flatten([

stacks/dev/hetzner/terragrunt.hcl

@@ -22,17 +22,26 @@ include {
 
 inputs = {
   k3s_manager_pool = {
-    count = 3
+    count = 1
   }
   k3s_worker_pools = [
-    {
-      count = 2
-      name  = "pool1"
-    },
-    {
-      count = 1
-      name  = "pool2"
-    }
+    // {
+    //   count = 2
+    //   name  = "pool1"
+    // },
+    // {
+    //   count = 1
+    //   name  = "pool2"
+    // }
   ]
+  k3s_existing_worker_pools = {
+    "pool1" : [
+      {
+        name = "homelab-001"
+        host = "192.168.1.243"
+        user = "homelab"
+      }
+    ]
+  }
   network_subnet = "10.2.0.0/16"
 }