From 9784eccdef3667ad29dd74b7983772eb011120b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Tue, 25 Jun 2024 17:24:53 +0200 Subject: [PATCH] ci: Add initial CI --- .github/pipeline.yaml | 23 +++ .gitlab/.gitignore | 1 + .gitlab/container/Containerfile | 37 +++++ .gitlab/pipeline.yaml | 44 ++++++ .gitlab/scripts/before-script.sh | 11 ++ .gitlab/scripts/upload-images.sh | 39 +++++ justfile | 19 ++- tools/format-rust.sh | 15 ++ tools/general.sh | 186 +++++++++++++++++++++++ tools/lint-rust.sh | 23 +++ {nix => tools/nix}/flake.lock | 0 {nix => tools/nix}/flake.nix | 4 +- tools/start-gitlab-runner-docker.sh | 155 +++++++++++++++++++ tools/start-gitlab-runner-podman.sh | 223 ++++++++++++++++++++++++++++ 14 files changed, 774 insertions(+), 6 deletions(-) create mode 100644 .github/pipeline.yaml create mode 100644 .gitlab/.gitignore create mode 100644 .gitlab/container/Containerfile create mode 100644 .gitlab/pipeline.yaml create mode 100755 .gitlab/scripts/before-script.sh create mode 100755 .gitlab/scripts/upload-images.sh create mode 100755 tools/format-rust.sh create mode 100755 tools/general.sh create mode 100755 tools/lint-rust.sh rename {nix => tools/nix}/flake.lock (100%) rename {nix => tools/nix}/flake.nix (94%) create mode 100755 tools/start-gitlab-runner-docker.sh create mode 100755 tools/start-gitlab-runner-podman.sh diff --git a/.github/pipeline.yaml b/.github/pipeline.yaml new file mode 100644 index 0000000..4fe0479 --- /dev/null +++ b/.github/pipeline.yaml @@ -0,0 +1,23 @@ +name: rdf-protect + +on: [push] + +jobs: + trigger-gitlab: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Mirror + Trigger CI + uses: SvanBoxel/gitlab-mirror-and-ci-action@master + with: + args: "https://gitlab.com//" + env: + FOLLOW_TAGS: "false" + FORCE_PUSH: "true" + GITLAB_HOSTNAME: "gitlab.datascience.ch" + GITLAB_USERNAME: ${{ secrets.GITLAB_USERNAME }} + GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }} + GITLAB_PROJECT_ID: "454" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitlab/.gitignore b/.gitlab/.gitignore new file mode 100644 index 0000000..4083037 --- /dev/null +++ b/.gitlab/.gitignore @@ -0,0 +1 @@ +local diff --git a/.gitlab/container/Containerfile b/.gitlab/container/Containerfile new file mode 100644 index 0000000..79f14c6 --- /dev/null +++ b/.gitlab/container/Containerfile @@ -0,0 +1,37 @@ +# This is a docker image containing docker and a Nix store. +# This enables to either run Docker images inside this one, +# or use `nix develop` to start a sandboxed environment to +# do other non-docker related stuff. + +FROM alpine:latest as base-podman +LABEL org.opencontainers.image.source https://github.com/sdsc-ordes/rdf-protect +LABEL org.opencontainers.image.description "CI container image for rdf-protect" +LABEL org.opencontainers.image.license "Apache" + +RUN apk add findutils coreutils git jq curl bash just parallel podman + +# Nix Image +# =============================================== +FROM base-podman as ci-nix +RUN [ "TARGETPLATFORM" = "linux/amd64" ] || echo "Platform not yet supported." +COPY ./tools /container-setup + +# Install Nix and pre-cache the env. +RUN bash -c ". /container-setup/general.sh && ci_setup_nix" +COPY rust-toolchain.toml /container-setup/ +RUN cd /container-setup && \ + git init && git add . && \ + nix --accept-flake-config \ + build --no-link "./tools/nix#devShells.x86_64-linux.ci" + +# Format image. +# =============================================== +FROM ci-nix as ci-format + +# Lint image. +# =============================================== +FROM ci-nix as ci-lint + +# Build image. +# =============================================== +FROM ci-nix as ci-build diff --git a/.gitlab/pipeline.yaml b/.gitlab/pipeline.yaml new file mode 100644 index 0000000..7ff4d84 --- /dev/null +++ b/.gitlab/pipeline.yaml @@ -0,0 +1,44 @@ +stages: + - lint + - format + - build + +.defaults-rules: &defaults-rules + - if: "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature|bugfix/ || + $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" + when: always + +.main-rules: &main-rules + - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" + when: always + +lint: + stage: lint + needs: [] + image: ghcr.io/sdsc-ordes/rdf-protect:ci-lint-1.0.0 + rules: + - *defaults-rules + script: + - source .gitlab/scripts/before-script.sh + - nix develop .#ci --command just lint + +format: + stage: format + needs: [] + image: ghcr.io/sdsc-order/rdf-protect:ci-format-1.0.0 + rules: + - *defaults-rules + script: + - source .gitlab/scripts/before-script.sh + - nix develop .#ci --command just format-general + - nix develop .#ci --command just format + +build: + stage: build + needs: [] + image: ghcr.io/sdsc-ordes/rdf-protect:ci-build-1.0.0 + rules: + - *defaults-rules + script: + - source .gitlab/scripts/before-script.sh + - nix develop .#ci --command just build diff --git a/.gitlab/scripts/before-script.sh b/.gitlab/scripts/before-script.sh new file mode 100755 index 0000000..8bd981d --- /dev/null +++ b/.gitlab/scripts/before-script.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# This script is sourced. +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +ci_container_mgr_setup + +unset ROOT_DIR diff --git a/.gitlab/scripts/upload-images.sh b/.gitlab/scripts/upload-images.sh new file mode 100755 index 0000000..18c5f8b --- /dev/null +++ b/.gitlab/scripts/upload-images.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +function build_ci_image() { + local image_type="$1" + local repository="$2" + local tag="$image_type-$3" + + local image_name="$repository:$tag" + + print_info "Building image '$image_name'." + + ci_container_mgr build -f "$container_file" \ + --target "$image_type" \ + -t "$image_name" \ + . || die "Could not build image." + + ci_container_mgr push -f "$image_name" || die "Could not upload image." +} + +repository="${1:-ghcr.io/sdsc-ordes/rdf-protect}" +tag="${2:-1.0.0}" +container_file=".gitlab/container/Containerfile" + +if [ "${CI:-}" = "true" ]; then + ci_container_mgr_login "$DOCKER_REPOSITORY_READ_USERNAME" "$DOCKER_REPOSITORY_READ_TOKEN" +fi + +readarray -t images < <(grep -E "as ci-.*" "$container_file" | sed -E 's@.*as (ci-.*)$@\1@g') +for image in "${images[@]}"; do + build_ci_image "$image" "$repository" "$tag" +done diff --git a/justfile b/justfile index 4b357bb..f80ea28 100644 --- a/justfile +++ b/justfile @@ -9,7 +9,7 @@ container_mgr := "podman" # Enter a Nix development shell. nix-develop: - cd "{{root_dir}}" && nix develop ./nix#default + cd "{{root_dir}}" && nix develop ./tools/nix#default # Build the executable. build *args: @@ -23,7 +23,18 @@ watch: run: cd "{{root_dir}}" && cargo run "${@:1}" -format: +format-general *args: + # Not implemented yet. + true + +format *args: + cd "{{comp_dir}}" && \ + "{{root_dir}}/tools/format-rust.sh" {{args}} + +lint *args: + cd "{{comp_dir}}" && \ + "{{root_dir}}/tools/lint-rust.sh" {{args}} + +upload-ci-images: cd "{{root_dir}}" && \ - {{container_mgr}} run -v "{{root_dir}}:/repo" -v "$(pwd):/workspace" -w "/workspace" \ - instrumentisto/rust:nightly-alpine cargo fmt -- --config-path /repo + .gitlab/scripts/upload-images.sh diff --git a/tools/format-rust.sh b/tools/format-rust.sh new file mode 100755 index 0000000..5e441a9 --- /dev/null +++ b/tools/format-rust.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +print_info "Run Rust format." +ci_wrap_container \ + ghcr.io/sdsc-ordes/rdf-protect:ci-format-1.0.0 \ + nix develop ./tools/nix#ci --command \ + cargo fmt "$@" diff --git a/tools/general.sh b/tools/general.sh new file mode 100755 index 0000000..c716861 --- /dev/null +++ b/tools/general.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# shellcheck disable=SC2154,SC2086 + +function _print() { + local color="$1" + local flags="$2" + local header="$3" + shift 3 + + local hasColor="0" + if [ "${FORCE_COLOR:-}" != 1 ]; then + [ -t 1 ] && hasColor="1" + else + hasColor="1" + fi + + if [ "$hasColor" = "0" ] || [ "${LOG_COLORS:-}" = "false" ]; then + local msg + msg=$(printf '%b\n' "$@") + msg="${msg//$'\n'/$'\n' }" + echo $flags -e "-- $header$msg" + else + local s=$'\033' e='[0m' + local msg + msg=$(printf "%b\n" "$@") + msg="${msg//$'\n'/$'\n' }" + echo $flags -e "${s}${color}-- $header$msg${s}${e}" + fi +} +function print_info() { + _print "[0;94m" "" "" "$@" +} + +function print_warning() { + _print "[0;31m" "" "WARN: " "$@" >&2 +} + +function print_error() { + _print "[0;31m" "" "ERROR: " "$@" >&2 +} + +function die() { + print_error "$@" + exit 1 +} + +function ci_is_running() { + if [ "${CI:-}" = "true" ]; then + return 0 + fi + + return 1 +} + +function ci_wrap_container() { + local container="$1" + shift 1 + local cmd=("$@") + + if [ "$OSTYPE" = "nixos" ]; then + "${cmd[@]}" + else + ci_container_mgr_run_mounted "$(pwd)" "$container" "${cmd[@]}" + fi +} + +function ci_setup_githooks() { + local installPrefix="${1:-$CI_BUILDS_DIR/githooks}" + mkdir -p "$installPrefix" + + print_info "Install Githooks in '$installPrefix'." + githooks-cli installer --non-interactive --prefix "$installPrefix" + + git hooks config enable-containerized-hooks --global --set + git hooks config container-manager-types --global --set "podman,docker" + + print_info "Pull all shared Githooks repositories." + git hooks shared update + + export CI_GITHOOKS_INSTALL_PREFIX="$installPrefix" +} + +function ci_setup_nix() { + local install_prefix="${1:-/usr/sbin}" + + print_info "Install Nix." + apk add curl bash xz shadow + sh <(curl -L https://nixos.org/nix/install) --daemon --yes + cp /root/.nix-profile/bin/* "$install_prefix/" + + print_info "Enable Features for Nix." + mkdir -p ~/.config/nix + { + echo "experimental-features = nix-command flakes" + echo "accept-flake-config = true" + } >~/.config/nix/nix.conf +} + +# Run the container manager which is defined. +function ci_container_mgr() { + if command -v podman &>/dev/null; then + echo -e "Running podman as:\n$(printf "'%s' " "podman" "$@")" >&2 + podman "$@" + else + echo -e "Running docker as:\n$(printf "'%s' " "docker" "$@")" + docker "$@" + fi +} + +# Define the container id `CI_JOB_CONTAINER_ID` where +# this job runs. Useful to mount same volumes as in +# this container with `ci_run_podman`. +function ci_container_mgr_setup() { + export CONTAINER_HOST="unix://var/run/podman.sock" + print_info "Container host: '$CONTAINER_HOST'" + + job_container_id=$(ci_container_mgr ps \ + --filter "label=com.gitlab.gitlab-runner.type=build" \ + --filter "label=com.gitlab.gitlab-runner.job.id=$CI_JOB_ID" \ + --filter "label=com.gitlab.gitlab-runner.project.id=$CI_PROJECT_ID" \ + --filter "label=com.gitlab.gitlab-runner.pipeline.id=$CI_PIPELINE_ID" \ + --format "{{ .ID }}") || + die "Could not find 'build' container for job id: '$CI_JOB_ID'." + + [ -n "$job_container_id" ] || die "Job id is empty." + + export CI_JOB_CONTAINER_ID="$job_container_id" + print_info "Job container id: '$CI_JOB_CONTAINER_ID'" +} + +function ci_container_mgr_login() { + local user="$1" + local token="$2" + + [ -n "$token" ] || die "Docker login token is empty" + echo "$token" | + ci_container_mgr login --password-stdin --username "$user" || + die "Could not log into docker." +} + +# Run container mgr. In CI with volume mount from the +# current build container `CI_JOB_CONTAINER_ID`. +function ci_container_mgr_run() { + if ci_is_running; then + ci_container_mgr run --volumes-from "$CI_JOB_CONTAINER_ID" "$@" + else + ci_container_mgr run "$@" + fi +} + +function ci_container_mgr_run_mounted() { + local repo workspace_rel in_cmd + repo=$(git rev-parse --show-toplevel) + workspace_rel=$(cd "$1" && pwd) + workspace_rel=$(realpath --relative-to "$repo" "$workspace_rel") + + shift 1 + in_cmd=("$@") + + local mnt_args=() + local cmd=() + + if ! ci_is_running; then + cmd=("${in_cmd[@]}") + mnt_args+=(-v "$repo:/repo") + mnt_args+=(-w "/repo/$workspace_rel") + else + # Not needed to mount anything, since already existing + # under the same path as `repo`. + # + # All `/repo` and `/workspace` paths in + # command given are replaced with correct + # paths to mounted volume in CI + for arg in "${in_cmd[@]}"; do + cmd+=("$(echo "$arg" | + sed -E \ + -e "s@/workspace@$workspace_rel@g" \ + -e "s@/repo@$repo@g")") + done + + mnt_args+=(-w "$repo/$workspace_rel") + fi + + ci_container_mgr_run "${mnt_args[@]}" "${cmd[@]}" +} diff --git a/tools/lint-rust.sh b/tools/lint-rust.sh new file mode 100755 index 0000000..c670387 --- /dev/null +++ b/tools/lint-rust.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +set -e +set -u + +ROOT_DIR=$(git rev-parse --show-toplevel) +. "$ROOT_DIR/tools/general.sh" + +cd "$ROOT_DIR" + +print_info "Run Rust Clippy linter." +ci_wrap_container \ + ghcr.io/sdsc-ordes/rdf-protect:ci-lint-1.0.0 \ + nix develop ./tools/nix#ci --command \ + cargo clippy --no-deps -- -A clippy::needless_return "$@" || + die "Rust clippy failed." + +print_info "Run Rust Miri to check undefined behaviour." +ci_wrap_container \ + ghcr.io/sdsc-ordes/rdf-protect:ci-lint-1.0.0 \ + nix develop ./tools/nix#ci --command \ + cargo miri test "$@" || + die "Rust Miri failed." diff --git a/nix/flake.lock b/tools/nix/flake.lock similarity index 100% rename from nix/flake.lock rename to tools/nix/flake.lock diff --git a/nix/flake.nix b/tools/nix/flake.nix similarity index 94% rename from nix/flake.nix rename to tools/nix/flake.nix index 7f42e5e..bbe473b 100644 --- a/nix/flake.nix +++ b/tools/nix/flake.nix @@ -57,7 +57,7 @@ }; # Set the rust toolchain from the `rust-toolchain.toml`. - rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ../../rust-toolchain.toml; # Things needed only at compile-time. nativeBuildInputsBasic = with pkgs; [ @@ -84,7 +84,7 @@ ci = mkShell { inherit buildInputs; - nativeBuildInputs = nativeBuildInputsBasic; + nativeBuildInputs = nativeBuildInputsBasic ++ nativeBuildInputsDev; }; }; } diff --git a/tools/start-gitlab-runner-docker.sh b/tools/start-gitlab-runner-docker.sh new file mode 100755 index 0000000..c52ed8e --- /dev/null +++ b/tools/start-gitlab-runner-docker.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091 +# +# Create a gitlab runner (docker executor) +# by first visiting `CI/CD Settings` page and +# creating a `linux` runner which gives you a `` needed +# for this script. +# +# This creates a docker container which runs the Gitlab runner +# which will execute jobs over the `docker` executor. +# The running container is not that safe in the sense that the Docker socket +# is mounted into the container (privilege escalation can be done: +# - https://blog.nestybox.com/2020/10/21/gitlab-dind.html +# - https://github.com/stealthcopter/deepce). +# +# TODO: This script should use the runtime `sysbox-runc` for better isolation. +# So far its not available on NixOS. +# https://github.com/NixOS/nixpkgs/issues/271901 +# +# The `gitlab-runner` does not forward the socket to the job containers +# because that would be to risky. Nevertheless, +# docker-in-docker for a job works as shown below. +# +# Usage: +# ```shell +# start-gitlab-runner-docker.sh [--force] [] +# ``` +# Read token from stdin. + +# Usage in Pipeline: +# +# A job which uses `docker` to run/build images. +# the `service`-container `docker:24-dind`. +# +# ```yaml +# docker-run-build: +# image: docker:24 +# # +# # When you use the dind service, you must instruct Docker to talk with +# # the daemon started inside of the service 'docker:*-dind'. +# # The daemon is available with a network connection instead of the default +# # /var/run/docker.sock socket. +# # Docker does this automatically by setting the DOCKER_HOST in +# # https://github.com/docker-library/docker/blob/master/docker-entrypoint.sh#L30 +# # The 'docker' hostname is the alias of the service container as described +# # at https://docs.gitlab.com/ee/ci/services/#accessing-the-services. +# # which is `docker` and then DOCKER_HOST=tcp://docker:2376 +# services: +# - docker:24-dind +# +# script: +# - docker info +# - docker run alpine:latest cat /etc/os-release +# - docker build -f Dockerfile . +# ``` + +set -e +set -u + +ROOT=$(git rev-parse --show-toplevel) +. "$ROOT/tools/general.sh" + +force="false" +max_jobs=4 +config_dir="$ROOT/.gitlab/local/config" +runner_name="gitlab-runner-md2pdf-docker" +cores=$(grep "^cpu\\scores" /proc/cpuinfo | uniq | cut -d ' ' -f 3) + +function modify_config() { + local key="$1" + local value="$2" + local type="${3:-json}" + + docker run --rm -v "$config_dir/config.toml:/config.toml" \ + "ghcr.io/tomwright/dasel" put -f /config.toml \ + -t "$type" \ + -s "$key" \ + -v "$value" || + die "Could not set gitlab runner config key '$key' to '$value'" +} + +function create() { + local token="${1:-}" + + if [ "$token" = "-" ] || [ -z "$token" ]; then + read -rs -p "Enter Gitlab Runner Token: " token || + die "Could not read token from TTY." + fi + + rm -rf "$config_dir" >/dev/null || true + mkdir -p "$config_dir" + + docker run -d \ + --cpus "$cores" \ + --name "$runner_name" \ + --restart always \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$config_dir":/etc/gitlab-runner \ + gitlab/gitlab-runner:latest || die "Could not create gitlab-runner" + + docker exec -it "$runner_name" gitlab-runner register \ + --non-interactive \ + --url "https://gitlab.com" \ + --token "$token" \ + --executor docker \ + --description "$runner_name" \ + --docker-image "alpine:latest" \ + --docker-privileged \ + --docker-volumes "/certs/client" || die "Could not start gitlab runner" + + modify_config ".concurrent" "$max_jobs" + modify_config ".runners.first().docker.pull_policy" \ + '["always", "if-not-present"]' + + docker exec -it "$runner_name" gitlab-runner start || die "Could not start runner." +} + +function stop() { + if is_running; then + print_info "Stop runner '$runner_name' ..." + docker stop "$runner_name" + + fi + + if is_exited; then + # shellcheck disable=SC2046 + docker rm $(docker ps -a -q) + fi +} + +function is_running() { + [ "$(docker inspect -f '{{.State.Status}}' "$runner_name" 2>/dev/null || true)" = 'running' ] || return 1 + return 0 +} + +function is_exited() { + [ "$(docker inspect -f '{{.State.Status}}' "$runner_name" 2>/dev/null || true)" = 'exited' ] || return 1 + return 0 +} + +if [ "${1:-}" = "--force" ]; then + force="true" + shift 1 +fi + +if [ "$force" = "true" ]; then + stop +fi + +if ! is_running; then + create "$@" +else + print_info "Gitlab runner '$runner_name' is already running. Restart it." + docker restart "$runner_name" || die "Could not restart gitlab runner" +fi diff --git a/tools/start-gitlab-runner-podman.sh b/tools/start-gitlab-runner-podman.sh new file mode 100755 index 0000000..efc9ae8 --- /dev/null +++ b/tools/start-gitlab-runner-podman.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090,SC1091,SC2015 +# +# Create a gitlab runner (docker executor) +# by first visiting `CI/CD Settings` page and +# creating a `linux` runner which gives you a `` needed +# for this script. +# +# This creates a container which runs the Gitlab runner +# which will execute jobs over the `podman` executor. +# +# This container is based on [`pipglr`](https://gitlab.com/qontainers/pipglr) +# which uses two container volumes (`pipglr-storage` and `pipglr-cache`) +# which are attached and contains +# `podman` (user `podman`) and `gitlab-runner` (user `runner`) +# to have a rootless container experience which provides much more security. +# The two volumes contain the images created/built by CI. The volumes can safely +# be wiped if space is needed. +# +# The podman socket created inside this container +# will be mounted to each job container the `gitlab-runner` creates. +# This makes also use of caching directly possible which is cool. +# +# Usage: +# ```shell +# start-gitlab-runner-docker.sh [--force] [] +# ``` +# Read token from stdin. +# ```shell +# start-gitlab-runner-docker.sh [--force] - +# +# Usage in Pipeline: +# +# A job which uses `podman` (linked to podman) to run/build images. +# The gitlab-runner cannot serve `services` statements as in the +# `start-gitlab-runner-docker.sh` (uses `docker` +# `--link` which is anyway deprecated) +# +# ```yaml +# podman-remote-run-build: +# image: quay.io/podman/stable:latest +# variables: +# CONTAINER_HOST: unix://var/run/docker.sock +# script: +# - podman info +# - podman run alpine:latest cat /etc/os-release +# - podman build -f Dockerfile . +# ``` +# +# The following (custom build image) also works: +# +# ```yaml +# podman-remote-alpine-run-build: +# image: alpine:latest +# variables: +# CONTAINER_HOST: unix://var/run/docker.sock +# script: +# - apk add podman +# - podman info +# - podman run alpine:latest cat /etc/os-release +# - podman build -f Dockerfile . +# ``` + +set -e +set -u + +ROOT=$(git rev-parse --show-toplevel) +. "$ROOT/tools/general.sh" + +force="false" +max_jobs=4 +config_dir="$ROOT/.gitlab/local/config" +runner_name="gitlab-runner-md2pdf-podman" +cores=$(grep "^cpu\\scores" /proc/cpuinfo | uniq | cut -d ' ' -f 3) +# image="registry.gitlab.com/qontainers/pipglr:latest" +image="pipglr:dev-latest-alpine" + +function clean_up() { + if [ -f "$config_dir/config.toml" ]; then + rm -rf "$config_dir/config.toml" + fi +} + +trap clean_up EXIT +function modify_config() { + local key="$1" + local value="$2" + local type="${3:-json}" + + podman run --rm -v "$config_dir/config.toml:/config.toml" \ + "ghcr.io/tomwright/dasel" put -f /config.toml \ + -t "$type" \ + -s "$key" \ + -v "$value" || + die "Could not set gitlab runner config key '$key' to '$value'" +} + +function register_runner() { + print_info "Registering gitlab-runner ..." + local token="${1:-}" + + if [ "$token" = "-" ] || [ -z "$token" ]; then + read -rs -p "Enter Gitlab Runner Token: " token || + die "Could not read token from stdin." + fi + + podman secret rm REGISTRATION_TOKEN &>/dev/null || true + echo "$token" | podman secret create REGISTRATION_TOKEN - || + die "Could not set registration token secret." + + # Register Gitlab runner. + (cd "$config_dir" && + touch config.toml && + podman container runlabel register "$image") || + die "Could not register gitlab-runner." + + # Modify Gitlab runner config. + modify_config ".concurrent" "$max_jobs" + modify_config ".runners.first().docker.pull_policy" \ + '["always"]' + modify_config ".runners.first().docker.volumes.append()" \ + "/home/runner/podman.sock:/var/run/podman.sock:rw" string + + # Add an auxiliary volume `auxvol`. + modify_config ".runners.first().docker.volumes.append()" \ + "auxvol:/auxvol" string + + modify_config ".runners.first().pre_build_script" \ + "echo 'Prebuild'\\nenv" string + + podman secret rm config.toml &>/dev/null || true + podman secret create config.toml "$config_dir/config.toml" || + die "Could not create config.toml secret." + + print_info "Config file:" \ + "$(sed 's/token\s*=.*/token = ***/g' "$config_dir/config.toml")" + + rm "$config_dir/config.toml" +} + +function assert_volumes() { + print_info "Asserting needed volumes ..." + + local volumes + volumes=$(podman volume list --format="{{ .Name }}") || + die "Could not get volumes" + + if ! echo "$volumes" | grep -q "pipglr-storage"; then + podman container runlabel setupstorage "$image" + fi + + if ! echo "$volumes" | grep -q "pipglr-cache"; then + podman container runlabel setupcache "$image" + fi +} + +function start_runner() { + print_info "Start runner '$runner_name' ..." + + # Run the Gitlab runner. We cannot user `podman container runlabel run "$image"` + # because we need to set some cpu constraints. + podman run -dt --name "$runner_name" \ + --cpus "$cores" \ + --secret config.toml,uid=1001,gid=1001 \ + -v pipglr-storage:/home/podman/.local/share/containers \ + -v pipglr-cache:/cache \ + --systemd true --privileged \ + --device /dev/fuse "$image" + + podman exec -it --user root "$runner_name" \ + bash -c "mkdir -p /etc/containers; + cp /usr/share/containers/seccomp.json /etc/containers/seccomp.json" +} + +function create() { + rm -rf "$config_dir" >/dev/null || true + mkdir -p "$config_dir" + + register_runner "$@" + assert_volumes + + start_runner +} + +function stop() { + if is_running; then + print_info "Stop runner '$runner_name' ..." + podman stop "$runner_name" + + fi + + if is_exited; then + # shellcheck disable=SC2046 + podman rm $(podman ps -a -q) + fi +} + +function is_running() { + [ "$(podman inspect -f '{{.State.Status}}' "$runner_name" 2>/dev/null || true)" = 'running' ] || return 1 + return 0 +} + +function is_exited() { + [ "$(podman inspect -f '{{.State.Status}}' "$runner_name" 2>/dev/null || true)" = 'exited' ] || return 1 + return 0 +} + +if [ "${1:-}" = "--force" ]; then + force="true" + shift 1 +fi + +if [ "$force" = "true" ]; then + stop +fi + +if ! is_running; then + create "$@" +else + print_info "Gitlab runner '$runner_name' is already running. Restart it." + podman restart "$runner_name" || + die "Could not restart gitlab runner '$runner_name'." +fi