diff --git a/.github/workflows/pkg-buildbot-builder.yml b/.github/workflows/pkg-buildbot-builder.yml new file mode 100644 index 00000000..e6a105a4 --- /dev/null +++ b/.github/workflows/pkg-buildbot-builder.yml @@ -0,0 +1,23 @@ +name: buildbot-builder Service Container +on: + workflow_dispatch: + pull_request: + branches: + - master + paths: + - services/pkg/buildbot-builder/** + push: + branches: + - master + paths: + - services/pkg/buildbot-builder/** + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + main: + uses: ./.github/workflows/build-pkg.yml + with: + service_name: buildbot-builder diff --git a/.github/workflows/pkg-buildbot.yml b/.github/workflows/pkg-buildbot.yml new file mode 100644 index 00000000..905daa8f --- /dev/null +++ b/.github/workflows/pkg-buildbot.yml @@ -0,0 +1,23 @@ +name: buildbot Service Container +on: + workflow_dispatch: + pull_request: + branches: + - master + paths: + - services/pkg/buildbot/** + push: + branches: + - master + paths: + - services/pkg/buildbot/** + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + main: + uses: ./.github/workflows/build-pkg.yml + with: + service_name: buildbot diff --git a/services/pkg/buildbot-builder/Dockerfile b/services/pkg/buildbot-builder/Dockerfile new file mode 100644 index 00000000..a9376ef4 --- /dev/null +++ b/services/pkg/buildbot-builder/Dockerfile @@ -0,0 +1,27 @@ +FROM ghcr.io/void-linux/void-glibc-full:20240526R1 AS build + +RUN xbps-install -Suy xbps && xbps-install -uy git go +ARG GO_XBPS_SRC_VERSION=4c394976975f7e84062f56d437b41a0d48043792 +RUN env GOBIN=/usr/local/bin GOMODCACHE=/tmp/go \ + go install -v github.com/Duncaen/go-xbps-src/cmd/xbps-src-make@$GO_XBPS_SRC_VERSION + +FROM build AS buildbot-builder +RUN xbps-install -Suy xbps && xbps-install -uy python3 tini git curl bash make rsync +COPY --from=build /usr/local/bin/xbps-src-make /usr/local/bin/xbps-src-make +RUN rm -rf /var/cache/xbps && \ + groupadd --gid 418 void-builder && \ + useradd --uid 418 --gid 418 -G xbuilder -M -d /buildbot void-builder && \ + mkdir /venv /buildbot && \ + chown void-builder:void-builder /venv /buildbot +USER void-builder +# version must be synced with ../buildbot +RUN python3 -m venv /venv && \ + /venv/bin/pip3 install wheel && \ + /venv/bin/pip3 install 'buildbot-worker~=4.0.0' +WORKDIR /buildbot +RUN mkdir -p /buildbot && \ + ln -sf /local/info /buildbot/info && \ + ln -sf /local/xbps-src.conf /buildbot/.xbps-src.conf +COPY buildbot.tac /buildbot/ +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/venv/bin/buildbot-worker", "start", "--nodaemon", "/buildbot"] diff --git a/services/pkg/buildbot-builder/buildbot.tac b/services/pkg/buildbot-builder/buildbot.tac new file mode 100644 index 00000000..9f95d197 --- /dev/null +++ b/services/pkg/buildbot-builder/buildbot.tac @@ -0,0 +1,56 @@ +# vim: set filetype=python: + +import os +import configparser + +from buildbot_worker.bot import Worker + +from twisted.application import service +from twisted.python.logfile import LogFile +from twisted.python.log import ILogObserver, FileLogObserver + +basedir = '.' +rotateLength = 10000000 +maxRotatedFiles = 10 + +# if this is a relocatable tac file, get the directory containing the TAC +if basedir == '.': + import os.path + basedir = os.path.abspath(os.path.dirname(__file__)) + +# note: this line is matched against to check that this is a worker +# directory; do not edit it. +application = service.Application('buildbot-worker') + +logfile = LogFile.fromFullPath( + os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) +application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + +ini = configparser.ConfigParser() +ini.read_dict({'buildbot': {}, 'worker': {}}) +ini.read('/local/config.ini') + +buildmaster_host = ini['buildbot'].get('host', 'localhost') +port = ini['buildbot'].getint('worker-port', 9989) +workername = ini['worker'].get('name', 'example-worker') +with open("/secrets/buildbot/worker-password") as f: + passwd = f.read() +keepalive = ini['worker'].getint('keepalive', 600) +umask = None +maxdelay = ini['worker'].getint('maxdelay', 300) +numcpus = None +allow_shutdown = None +maxretries = None +use_tls = 0 +delete_leftover_dirs = False +proxy_connection_string = None +protocol = 'pb' + +s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown, + maxRetries=maxretries, protocol=protocol, useTls=use_tls, + delete_leftover_dirs=delete_leftover_dirs, + proxy_connection_string=proxy_connection_string) +s.setServiceParent(application) diff --git a/services/pkg/buildbot/Dockerfile b/services/pkg/buildbot/Dockerfile new file mode 100644 index 00000000..5227d437 --- /dev/null +++ b/services/pkg/buildbot/Dockerfile @@ -0,0 +1,19 @@ +FROM ghcr.io/void-linux/void-glibc-full:20240526R1 + +RUN xbps-install -Suy xbps && xbps-install -uy python3 tini git curl +ARG THEME_VERSION=v0.1.0 +# version must be synced with ../buildbot-worker +RUN python3 -m venv /venv && \ + /venv/bin/pip3 install wheel && \ + /venv/bin/pip3 install 'buildbot[tls,bundle]~=4.0.0' treq \ + buildbot-prometheus buildbot-netauth \ + git+https://github.com/void-linux/buildbot-theme.git@$THEME_VERSION +WORKDIR /buildbot +COPY buildbot.tac /buildbot/ +COPY run.sh / +# buildbot config should be in its basedir +# buildbot upgrade-master hardcodes master.cfg +RUN ln -sf /local/buildbot.cfg /buildbot/buildbot.cfg && \ + ln -sf /buildbot/buildbot.cfg /buildbot/master.cfg +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/bin/sh", "/run.sh"] diff --git a/services/pkg/buildbot/buildbot.tac b/services/pkg/buildbot/buildbot.tac new file mode 100644 index 00000000..c0cd1f86 --- /dev/null +++ b/services/pkg/buildbot/buildbot.tac @@ -0,0 +1,20 @@ +# vim: set filetype=python: + +import os +import sys + +from twisted.application import service +from twisted.python.log import FileLogObserver +from twisted.python.log import ILogObserver + +from buildbot.master import BuildMaster + +basedir = os.path.abspath(os.path.dirname(__file__)) + +# note: this line is matched against to check that this is a buildmaster +# directory; do not edit it. +application = service.Application('buildmaster') +application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) + +m = BuildMaster(basedir, 'buildbot.cfg', umask=None) +m.setServiceParent(application) diff --git a/services/pkg/buildbot/run.sh b/services/pkg/buildbot/run.sh new file mode 100644 index 00000000..c32dff19 --- /dev/null +++ b/services/pkg/buildbot/run.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +: "${DB_UPGRADE:=$NOMAD_META_DB_UPGRADE}" + +if [ "$DB_UPGRADE" = "true" ]; then + until /venv/bin/buildbot upgrade-master /buildbot; do + echo "upgrading buildbot installation..." + sleep 1 + done +fi + +exec /venv/bin/buildbot start --nodaemon /buildbot diff --git a/services/pkg/docker-bake.hcl b/services/pkg/docker-bake.hcl index 228490d7..8d74c64a 100644 --- a/services/pkg/docker-bake.hcl +++ b/services/pkg/docker-bake.hcl @@ -25,6 +25,20 @@ target "infra-alps" { context = "services/pkg/alps/" } +target "infra-buildbot" { + inherits = ["_common"] + platforms = ["linux/amd64"] + args = { "LIBC" = "glibc" } + context = "services/pkg/buildbot/" +} + +target "infra-buildbot-builder" { + inherits = ["_common"] + platforms = ["linux/amd64"] + args = { "LIBC" = "glibc" } + context = "services/pkg/buildbot-builder/" +} + target "infra-debuginfod" { inherits = ["_common-musl"] context = "services/pkg/debuginfod/"