nomadic buildbot

docs/src/services/buildbot.md

@@ -2,13 +2,13 @@
 
 BuildBot is our legacy build scheduler.
 
-The buildbot master runs at
+The buildbot controller runs at
 [build.voidlinux.org](https://build.voidlinux.org) and provides
 unified scheduling to all other build tasks in the fleet.  BuildBot
 also exposes a web interface.
 
 The current status of the build infrastructure can be found on the
-build waterfall.  This view shows what each of the buildslaves is
+build waterfall.  This view shows what each of the builders is
 doing right now, and uses traffic light colors for build state.  A
 purple builder is usually a reason to contact void-ops and figure out
 what's wrong with the build host.
@@ -18,18 +18,44 @@ failed without needing to push a new commit.  Not all committers have
 access to restart failed builds this way.  If you believe that you
 should have this access, contact maldridge@.
 
-## Moving a buildslave
+## Moving a builder
 
-Don't.
+First, all builds need to be paused until the move can be completed.
+Builders must be moved in the groups they currently are in (glibc, musl,
+and aarch64). Sync the hostdir host volume to the new location, then
+migrate the nomad job to the new host.
 
-In the event that this is unavoidable, all builds need to be paused
-until the move can be completed.  In the even the builder that needs
-to be moved is on the musl cluster, all musl builders will need to be
-moved with it.  Similarly, the aarch64 builders must always move as a
-pair.
+## Updating buildbot
+
+The buildbot controller and builders should be kept at the same buildbot
+version.
+
+1. Update the buildbot version in both the `buildbot` and
+   `buildbot-builder` Dockerfiles.
+2. Rebuild the service containers by triggering a build for the
+   `infra-buildbot` and `infra-buildbot-builder` containers in CI.
+3. Update the nomad jobs to use the new containers and restart them
+   with the task meta variable `db-upgrade = "true"`. This will run
+   any necessary migrations on startup.
+4. Re-set `db-upgrade = "false"`. The job does not need to be
+   redeployed, but on the next deployment, it will not run the upgrade.
+
+## Future Plans
+
+Buildbot 4 uses the database to store logs. Currently, we are using the sqlite
+backend. If this causes performance issues in the future, we should switch to
+[postgres](https://docs.buildbot.net/current/manual/deploy.html#using-a-database-server).
+To migrate the database from sqlite, the steps appear to be:
+
+1. Ensure the sqlite database is fully upgraded by running the controller task with
+   `db-upgrade = "true"`.
+2. Set up a postgres server with a buildbot user (see Buildbot's documentation linked
+   above).
+3. Run `buildbot copydb postgres://buildbot:passw0rd@ip/buildbot /path/to/buildbot/basedir`.
+4. Update the buildbot configuration to use the new database url.
 
 ## EOL
 
-BuildBot is slated for replacement this fall/winter.  The system will
-be replaced by the Distributed XBPS Package Builder (dxpb) which will
-resolve many of the long standing problems in the buildbot.
+BuildBot is slated for replacement in the future.  The system will
+be replaced by the Distributed XBPS Package Builder (dxpb) or nbuild
+which will resolve many of the long standing problems in the buildbot.

services/nomad/build/buildbot-worker.nomad

@@ -0,0 +1,139 @@
+job "buildbot-worker" {
+  type = "service"
+  datacenters = ["VOID"]
+  namespace = "build"
+
+  dynamic "group" {
+    for_each = [
+      // memory is ~85% of capacity
+      { name = "glibc",   jobs = 10, mem = 109512 },
+      { name = "musl",    jobs = 6,  mem = 54512  },
+      { name = "aarch64", jobs = 3,  mem = 27256  },
+    ]
+    labels = [ "buildbot-worker-${group.value.name}" ]
+
+    content {
+      network {
+        mode = "bridge"
+        hostname = "void-buildbot-worker-${group.value.name}"
+      }
+
+      dynamic "volume" {
+        for_each = [ "${group.value.name}" ]
+        labels = [ "${volume.value}_hostdir" ]
+
+        content {
+          type = "host"
+          source = "${volume.value}_hostdir"
+          read_only = false
+        }
+      }
+
+      // https://github.com/hashicorp/nomad/issues/8892
+      task "prep-hostdir" {
+        driver = "docker"
+
+        config {
+          image = "ghcr.io/void-linux/void-glibc-full:20240526R1"
+          command = "chown"
+          args = ["-R", "418:418", "/hostdir"]
+        }
+
+        volume_mount {
+          volume = "${group.value.name}_hostdir"
+          destination = "/hostdir"
+        }
+
+        lifecycle {
+          hook = "prestart"
+        }
+      }
+
+      task "buildbot-worker" {
+        driver = "docker"
+
+        user = "void-builder"
+
+        config {
+          image = "ghcr.io/void-linux/infra-buildbot-builder:20240928R1"
+          force_pull = true
+          cap_add = ["sys_admin"]
+        }
+
+        resources {
+          cores = "${group.value.jobs}"
+          memory = "${group.value.mem}"
+        }
+
+        volume_mount {
+          volume = "${group.value.name}_hostdir"
+          destination = "/hostdir"
+        }
+
+        template {
+          data = <<EOF
+{{ range service "buildbot-worker" -}}
+[buildbot]
+host = {{ .Address }}
+worker-port = {{ .Port }}
+{{ end -}}
+
+[worker]
+name = worker-${group.value.name}
+EOF
+          destination = "local/config.ini"
+        }
+
+        template {
+          data = <<EOF
+XBPS_MAKEJOBS=${group.value.jobs}
+XBPS_CHROOT_CMD=uchroot
+XBPS_CHROOT_CMD_ARGS='-t'
+XBPS_CCACHE=yes
+XBPS_DEBUG_PKGS=yes
+XBPS_USE_GIT_REVS=yes
+XBPS_DISTFILES_MIRROR=https://sources.voidlinux.org
+XBPS_PRESERVE_PKGS=yes
+EOF
+          destination = "local/xbps-src.conf"
+        }
+
+        template {
+          data = "Void Build Operators <[email protected]>"
+          destination = "local/info/admin"
+        }
+
+        template {
+          data = <<EOF
+Void Linux Buildbot builder for ${group.value.name} running on {{ env "attr.unique.hostname" -}}
+EOF
+          destination = "local/info/host"
+        }
+
+        template {
+          data = <<EOF
+{{- with nomadVar "nomad/jobs/buildbot-worker" -}}
+{{ .worker_password }}
+{{- end -}}
+EOF
+          destination = "secrets/buildbot/worker-password"
+          perms = "400"
+          uid = 418
+          gid = 418
+        }
+
+        template {
+          data = <<EOF
+{{- with nomadVar "nomad/jobs/buildsync" -}}
+{{ .${group.value.name}_password }}
+{{- end -}}
+EOF
+          destination = "secrets/rsync/password"
+          perms = "400"
+          uid = 418
+          gid = 418
+        }
+      }
+    }
+  }
+}