From 53b67e3b0b56e64215b6abafe73e988936e3d18e Mon Sep 17 00:00:00 2001 From: Hiroki Sato Date: Sat, 26 Aug 2023 17:27:47 +0900 Subject: [PATCH 01/61] Use bookworm for building Docker image (Fixes #154) * Use Python venv to build domserver because Sphinx version should be 6.1.0 or higher to avoid a build issue. * Use PHP 8.x because it's default in Debian. * Use libcgoup2 instead of libcgroup1, which is no longer available. --- docker/domserver/Dockerfile | 11 +++++++---- docker/domserver/build.sh | 7 +++++-- docker/domserver/configure.sh | 2 +- docker/judgehost/Dockerfile | 4 ++-- docker/judgehost/Dockerfile.build | 2 +- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docker/domserver/Dockerfile b/docker/domserver/Dockerfile index 1abf03fd..7c92664d 100644 --- a/docker/domserver/Dockerfile +++ b/docker/domserver/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS domserver-build +FROM debian:bookworm-slim AS domserver-build LABEL org.opencontainers.image.authors="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive @@ -12,13 +12,16 @@ RUN apt update \ php-gd php-curl php-mysql php-json php-intl \ php-gmp php-xml php-mbstring \ sudo bsdmainutils ntp libcgroup-dev procps \ - python3-sphinx python3-sphinx-rtd-theme python3-pygments rst2pdf fontconfig python3-yaml \ + python3-venv fontconfig \ texlive-latex-recommended texlive-latex-extra \ - texlive-fonts-recommended texlive-lang-european latexmk \ + texlive-fonts-recommended texlive-lang-european latexmk tex-gyre \ libcurl4-gnutls-dev libjsoncpp-dev libmagic-dev \ enscript lpr ca-certificates \ && rm -rf /var/lib/apt/lists/* +# Use venv to install latest Sphinx. 6.1.0 or higher is required to build DOMjudge docs. +RUN python3 -m venv /venv && . /venv/bin/activate && pip3 install sphinx sphinx-rtd-theme rst2pdf + # Set up user RUN useradd -m domjudge @@ -35,7 +38,7 @@ COPY domserver/build.sh /domjudge-src/build.sh RUN /domjudge-src/build.sh # Now create an image with the actual build in it -FROM debian:bullseye-slim +FROM debian:bookworm-slim LABEL org.opencontainers.image.authors="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/domserver/build.sh b/docker/domserver/build.sh index 8b3c771f..0b61d0e0 100755 --- a/docker/domserver/build.sh +++ b/docker/domserver/build.sh @@ -1,9 +1,12 @@ #!/bin/sh -eu +# Use venv to use latest Sphinx. 6.1.0 or higher is required to build DOMjudge docs. +. /venv/bin/activate + cd /domjudge-src/domjudge* chown -R domjudge: . # If we used a local source tarball, it might not have been built yet -sudo -u domjudge make dist +sudo -u domjudge sh -c '. /venv/bin/activate && make dist' sudo -u domjudge ./configure -with-baseurl=http://localhost/ # Passwords should not be included in the built image. We create empty files here to prevent passwords from being generated. @@ -25,5 +28,5 @@ then rm /opt/domjudge/domserver/webapp/.env.local /opt/domjudge/domserver/webapp/.env.local.php fi -sudo -u domjudge make docs +sudo -u domjudge sh -c '. /venv/bin/activate && make docs' make install-docs diff --git a/docker/domserver/configure.sh b/docker/domserver/configure.sh index 55684046..5bd82143 100644 --- a/docker/domserver/configure.sh +++ b/docker/domserver/configure.sh @@ -23,7 +23,7 @@ done # Configure php -php_folder=$(echo "/etc/php/7."?"/") +php_folder=$(echo "/etc/php/8."?"/") php_version=$(basename "$php_folder") if [ ! -d "$php_folder" ] diff --git a/docker/judgehost/Dockerfile b/docker/judgehost/Dockerfile index 01ed7005..94b0a00d 100644 --- a/docker/judgehost/Dockerfile +++ b/docker/judgehost/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye +FROM debian:bookworm LABEL org.opencontainers.image.authors="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive \ @@ -14,7 +14,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt update \ && apt install --no-install-recommends --no-install-suggests -y \ dumb-init \ - acl zip unzip supervisor sudo procps libcgroup1 \ + acl zip unzip supervisor sudo procps libcgroup2 \ php-cli php-zip php-gd php-curl php-mysql php-json \ php-gmp php-xml php-mbstring python3 \ gcc g++ default-jre-headless default-jdk ghc fp-compiler \ diff --git a/docker/judgehost/Dockerfile.build b/docker/judgehost/Dockerfile.build index 0efdaf5b..9b373553 100644 --- a/docker/judgehost/Dockerfile.build +++ b/docker/judgehost/Dockerfile.build @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim +FROM debian:bookworm-slim LABEL org.opencontainers.image.authors="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive From cc9ad1211315a396074d59168cb622b5f5fbe5cd Mon Sep 17 00:00:00 2001 From: Hiroki Sato Date: Tue, 19 Sep 2023 20:15:02 +0900 Subject: [PATCH 02/61] Fix shellcheck --- docker/domserver/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/domserver/build.sh b/docker/domserver/build.sh index 0b61d0e0..418d9f9a 100755 --- a/docker/domserver/build.sh +++ b/docker/domserver/build.sh @@ -1,6 +1,7 @@ #!/bin/sh -eu # Use venv to use latest Sphinx. 6.1.0 or higher is required to build DOMjudge docs. +# shellcheck source=/dev/null . /venv/bin/activate cd /domjudge-src/domjudge* From 138ab77b5f32b27458b4170f6b1d7eafcda6b792 Mon Sep 17 00:00:00 2001 From: Hiroki Sato Date: Thu, 21 Sep 2023 23:58:18 +0900 Subject: [PATCH 03/61] Move venv activate line to make it clear what needs it --- docker/domserver/build.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/domserver/build.sh b/docker/domserver/build.sh index 418d9f9a..cfd41179 100755 --- a/docker/domserver/build.sh +++ b/docker/domserver/build.sh @@ -1,9 +1,5 @@ #!/bin/sh -eu -# Use venv to use latest Sphinx. 6.1.0 or higher is required to build DOMjudge docs. -# shellcheck source=/dev/null -. /venv/bin/activate - cd /domjudge-src/domjudge* chown -R domjudge: . # If we used a local source tarball, it might not have been built yet @@ -30,4 +26,7 @@ then fi sudo -u domjudge sh -c '. /venv/bin/activate && make docs' +# Use Python venv to use the latest Sphinx to build DOMjudge docs. +# shellcheck source=/dev/null +. /venv/bin/activate make install-docs From 65f98d7c35399f3f502d627a2c0eeaaab8f47fd4 Mon Sep 17 00:00:00 2001 From: Alireza Ghasemi Date: Mon, 9 Oct 2023 14:50:43 +0330 Subject: [PATCH 04/61] Fixup PHP version to 8.? Fix PHP version in the PHP timezone configuration script, used to access PHP's configuration directory. --- docker/domserver/scripts/start.d/10-timezone.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/domserver/scripts/start.d/10-timezone.sh b/docker/domserver/scripts/start.d/10-timezone.sh index ee8ace07..c046ad2a 100755 --- a/docker/domserver/scripts/start.d/10-timezone.sh +++ b/docker/domserver/scripts/start.d/10-timezone.sh @@ -6,7 +6,7 @@ dpkg-reconfigure -f noninteractive tzdata echo "[ok] Container timezone set to: ${CONTAINER_TIMEZONE}"; echo # Configure php -php_folder=$(echo "/etc/php/7."?"/") +php_folder=$(echo "/etc/php/8."?"/") cat > "$php_folder/fpm/conf.d/99-timezone.ini" <<- EOF [Date] From be8721b607f407839b5fad717a222c197b92cc9b Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 11 Nov 2023 15:12:20 +0100 Subject: [PATCH 05/61] Install `lsof` in judgehost containers Fix https://github.com/DOMjudge/domjudge-packaging/issues/159 --- docker-contributor/Dockerfile | 2 +- docker/judgehost/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index 85780c97..05fcbdac 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -24,7 +24,7 @@ RUN apt update \ nginx php8.1 php8.1-cli php8.1-fpm php8.1-zip \ php8.1-gd php8.1-curl php8.1-mysql php8.1-intl \ php8.1-gmp php8.1-xml php8.1-mbstring php8.1-xdebug php8.1-pcov \ - bsdmainutils ntp \ + bsdmainutils ntp lsof \ linuxdoc-tools linuxdoc-tools-text groff \ python3-sphinx python3-sphinx-rtd-theme python3-pip fontconfig python3-yaml \ texlive-latex-recommended texlive-latex-extra \ diff --git a/docker/judgehost/Dockerfile b/docker/judgehost/Dockerfile index 94b0a00d..0f06dcd9 100644 --- a/docker/judgehost/Dockerfile +++ b/docker/judgehost/Dockerfile @@ -14,7 +14,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt update \ && apt install --no-install-recommends --no-install-suggests -y \ dumb-init \ - acl zip unzip supervisor sudo procps libcgroup2 \ + acl lsof zip unzip supervisor sudo procps libcgroup2 \ php-cli php-zip php-gd php-curl php-mysql php-json \ php-gmp php-xml php-mbstring python3 \ gcc g++ default-jre-headless default-jdk ghc fp-compiler \ From c6b7e9db986ca4bae13ad7d640c858ec41f6f7de Mon Sep 17 00:00:00 2001 From: Alireza Ghasemi Date: Mon, 11 Dec 2023 15:51:49 +0330 Subject: [PATCH 06/61] Use --privileged instead of --cap-add=sys_admin due to build error Encountring the following errors with --cap-add=sys_admin inside the dj_make_chroot script; one inside the debootstrap, and the other somewhere inside the script itself: ... W: Failure trying to run: chroot "/chroot/domjudge" mount -t proc proc /proc W: See /chroot/domjudge/debootstrap/debootstrap.log for details ... mount: /chroot/domjudge/proc: cannot mount proc read-only. dmesg(1) may have more information after failed mount system call. ... The /chroot/domjudge/debootstrap/debootstrap.log does not exist nor the dmesg to investigate further (naive approach OFC). --- docker/build-judgehost.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build-judgehost.sh b/docker/build-judgehost.sh index 3e3691c3..e556c0bf 100755 --- a/docker/build-judgehost.sh +++ b/docker/build-judgehost.sh @@ -12,7 +12,7 @@ docker build -t "${docker_tag}-build" -f judgehost/Dockerfile.build . # Build chroot builder_name=$(echo "${docker_tag}" | sed 's/[^a-zA-Z0-9_-]/-/g') docker rm -f "${builder_name}" > /dev/null 2>&1 || true -docker run --name "${builder_name}" --cap-add=sys_admin "${docker_tag}-build" +docker run --name "${builder_name}" --privileged "${docker_tag}-build" docker cp "${builder_name}:/chroot.tar.gz" . docker cp "${builder_name}:/judgehost.tar.gz" . docker rm -f "${builder_name}" From 46d8f5a835e2e64a1d8c7f25b4038b711c36b282 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 24 Nov 2023 15:25:20 +0100 Subject: [PATCH 07/61] Integrate new PHP version for local development Make the image slightly smaller by removing the older PHP versions. Start using the latest version for faster PHP. In domjudge/domjudge CI we use the lowest supported one so we catch the whole spectrum. --- docker-contributor/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index 05fcbdac..7cc4faf8 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -12,8 +12,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ FPM_MAX_CHILDREN=40 \ DJ_SKIP_MAKE=0 \ DJ_DB_INSTALL_BARE=0 \ - PHPSUPPORTED="7.2 7.3 7.4 8.0 8.1 8.2" \ - DEFAULTPHPVERSION="8.1" \ + PHPSUPPORTED="8.0 8.1 8.2 8.3" \ + DEFAULTPHPVERSION="8.3" \ APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" # Install required packages and clean up afterwards to make this image layer smaller From a26b7a99197cf5a5e469aae44da5f7c3419f9a81 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Wed, 22 Feb 2023 20:22:56 +0100 Subject: [PATCH 08/61] Allow to specify UID/GID for contributor container. Fixes #143. --- docker-contributor/Dockerfile | 5 ++++- docker-contributor/README.md | 2 ++ docker-contributor/scripts/start.sh | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index 7cc4faf8..9878831e 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -4,6 +4,8 @@ LABEL maintainer="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive \ CONTAINER_TIMEZONE=Europe/Amsterdam \ + UID=1000 \ + GID=1000 \ MYSQL_HOST=mariadb \ MYSQL_USER=domjudge \ MYSQL_DATABASE=domjudge \ @@ -43,7 +45,8 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && mkdir -p /run/php # Set up users -RUN useradd -m domjudge \ +RUN groupadd -g $GID domjudge \ + && useradd -u $UID -g $GID -m domjudge \ && groupadd domjudge-run \ && for id in $(seq 0 4); do useradd -d /nonexistent -g nogroup -s /bin/false "domjudge-run-$id"; done diff --git a/docker-contributor/README.md b/docker-contributor/README.md index 3488b496..0f7212cc 100644 --- a/docker-contributor/README.md +++ b/docker-contributor/README.md @@ -57,6 +57,8 @@ bin/dj_setup_database install-examples The following environment variables are supported by the container: +* `UID` (defaults to `1000`): the ID of the user to run DOMjudge as. Should match the host OS user ID. +* `GID` (defaults to `1000`): the ID of the group to run DOMjudge as. Should match the host OS group ID. * `CONTAINER_TIMEZONE` (defaults to `Europe/Amsterdam`): allows you to change the timezone used inside the container. * `MYSQL_HOST` (defaults to `mariadb`): set the host to connect to for MySQL. Can be hostname or IP. Docker will add hostnames for any containers you `--link`, so in the example above, the MariaDB container will be available under the hostname `mariadb`. * `MYSQL_USER` (defaults to `domjudge`): set the user to use for connecting to MySQL. diff --git a/docker-contributor/scripts/start.sh b/docker-contributor/scripts/start.sh index 910f7da7..bf583e15 100644 --- a/docker-contributor/scripts/start.sh +++ b/docker-contributor/scripts/start.sh @@ -9,6 +9,11 @@ function file_or_env { fi } +echo "[..] Changing user/group ID" +groupmod -g "${GID}" domjudge +usermod -u "${UID}" domjudge +echo "[ok] User ID set to ${UID} and group ID set to ${GID}"; echo + echo "[..] Setting timezone" sudo ln -snf "/usr/share/zoneinfo/${CONTAINER_TIMEZONE}" /etc/localtime echo "${CONTAINER_TIMEZONE}" | sudo tee /etc/timezone From 2c412aed2745ee1f817af18a9e19a9e77204ba25 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Wed, 3 Jan 2024 15:02:38 +0100 Subject: [PATCH 09/61] Mention new UID/GID in README As mentiond by @christopher. --- docker-contributor/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-contributor/README.md b/docker-contributor/README.md index 0f7212cc..cd265fea 100644 --- a/docker-contributor/README.md +++ b/docker-contributor/README.md @@ -37,7 +37,7 @@ Next, if you are on Linux make sure you have cgroups enabled. See the [DOMjudge Now you can run DOMjudge itself using the following command: ```bash -docker run -v [path-to-domjudge-checkout]:[path-to-domjudge-checkout] -v /sys/fs/cgroup:/sys/fs/cgroup:ro --link dj-mariadb:mariadb -it -e PROJECT_DIR=[path-to-domjudge-checkout] -p 12345:80 --name domjudge --privileged domjudge/domjudge-contributor +docker run -v [path-to-domjudge-checkout]:[path-to-domjudge-checkout] -v /sys/fs/cgroup:/sys/fs/cgroup:ro --link dj-mariadb:mariadb -it -e UID="$(id -u)" -e GID="$(id -g)" -e PROJECT_DIR=[path-to-domjudge-checkout] -p 12345:80 --name domjudge --privileged domjudge/domjudge-contributor ``` Make sure you replace `[path-to-domjudge-checkout]` with the path to your local DOMjudge checkout. On recent macOS and Windows Docker builds, you should add `:cached` at the end of the volume (i.e. `-v [path-to-domjudge-checkout]:[path-to-domjudge-checkout]:cached`) to speed up the webserver a lot. From 12cb0807f59a3cb4a66cb9f1809f1800b5a28f44 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 6 Jan 2024 16:40:19 +0100 Subject: [PATCH 10/61] Removing breaking part from CI We fail on an error with libc-bin: https://gitlab.com/DOMjudge/domjudge-packaging/-/jobs/5876287000 I got this working in GitHub Actions already so we disable this here and fix this in GHA. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d94c460..ecea75bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ variables: timeout: 90 minutes parallel: matrix: - - ARCH: [amd64, arm64v8] + - ARCH: [amd64] .job_template: &ci_template script: From c51e4f7bd4eb36e05fd2db18b700df5c7366e441 Mon Sep 17 00:00:00 2001 From: MCJ Vasseur <14887731+vmcj@users.noreply.github.com> Date: Wed, 21 Sep 2022 19:11:09 +0200 Subject: [PATCH 11/61] Upgrade gitlabci container to ubuntu 22.04 (jammy) The default for PHP changes to 8.1. The package php8.1-json is now already provided by both php8.1-{fpm,cli} and became virtual. The npm pa11y tool is now install globally as npm changed its working, given that we run this in CI as either domjudge or root having it globally is actually better. --- .gitlab-ci.yml | 2 +- docker-gitlabci/Dockerfile | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecea75bf..46e447e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ stages: variables: FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true" # see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/1736 - GITLABCI_VERSION: "2.1" + GITLABCI_VERSION: "22.04" GITLABCI_IMAGE: "gitlabci" CONTRIBUTOR_IMAGE: "domjudge/domjudge-contributor" DOMJUDGE_VERSION: diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 6a07bb6e..81d43e70 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive ENV PHPSUPPORTED="7.2 7.3 8.0 8.1 8.2" ENV APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" @@ -7,7 +7,7 @@ RUN apt update && apt install -y \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ gcc g++ default-jre-headless default-jdk ghc fp-compiler libcgroup-dev \ devscripts shellcheck nginx libboost-regex-dev \ - php7.4 php7.4-cli php7.4-gd php7.4-curl php7.4-mysql php7.4-json php7.4-gmp php7.4-zip php7.4-xml php7.4-mbstring php7.4-fpm php7.4-intl php7.4-pcov \ + php8.1 php8.1-cli php8.1-gd php8.1-curl php8.1-mysql php8.1-gmp php8.1-zip php8.1-xml php8.1-mbstring php8.1-fpm php8.1-intl php8.1-pcov \ # W3c test \ httrack \ # Visual regression browser \ @@ -28,7 +28,7 @@ RUN apt update && apt install -y \ # Code coverage for unit test php-pear php-dev \ # Needed NPM packages \ - && npm install pa11y \ + && npm install -g pa11y \ # Needed python packages \ && pip install codespell \ && rm -rf /var/lib/apt/lists/* @@ -51,7 +51,7 @@ RUN add-apt-repository ppa:ondrej/php -y && apt update && \ for PACKAGE in $PACKAGES; do \ $APTINSTALL php${VERSION}-${PACKAGE#php*-}; \ done; \ - done && update-alternatives --set php /usr/bin/php7.4 + done && update-alternatives --set php /usr/bin/php8.1 # Put the gitlab user in sudo RUN echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers From 69e41f61df0c34270bf6f12540482652d0b5631e Mon Sep 17 00:00:00 2001 From: Michael <14887731+vmcj@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:05:08 +0100 Subject: [PATCH 12/61] Move creation of GitLab container to GitHub Actions Although we duplicate code now, the intent is so much easier to follow --- .../workflows/build-gitlab-container-PR.yml | 42 +++++ .../build-gitlab-container-release.yml | 40 +++++ .gitlab-ci.yml | 148 ------------------ 3 files changed, 82 insertions(+), 148 deletions(-) create mode 100644 .github/workflows/build-gitlab-container-PR.yml create mode 100644 .github/workflows/build-gitlab-container-release.yml delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml new file mode 100644 index 00000000..7ab62ea2 --- /dev/null +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -0,0 +1,42 @@ +name: 'Build GitLab CI container (PR)' + +on: + push: + pull_request_target: + branches: + - main + +jobs: + pr-gitlab: + if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'main' }} + name: PR GitLab image + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Use the same PHP config as the contributor containers + run: cp -r docker-{contributor,gitlabci}/php-config + + - name: Log in to GitHub Container registry (possibly at fork) + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ghcr.io/${{ github.repository }} + + - name: Build and push Docker images + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: "./docker-gitlabci" + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml new file mode 100644 index 00000000..62f049db --- /dev/null +++ b/.github/workflows/build-gitlab-container-release.yml @@ -0,0 +1,40 @@ +name: 'Build GitLab CI container (Accepted)' + +on: + push: + branches: + - main +jobs: + build-gitlab: + if: ${{ github.repository == domjudge/domjudge-packaging && github.ref == main }} + name: Build GitLab image + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Use the same PHP config as the contributor containers + run: cp -r docker-{contributor,gitlabci}/php-config + + - name: Log in to Docker Container registry + if: github.ref == 'refs/heads/main' + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: domjudge/gitlabci + + - name: Build and push Docker images + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: "./docker-gitlabci" + push: true + tags: "domjudge/gitlabci:22.04" + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 46e447e1..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,148 +0,0 @@ -include: '.tarball-release.yml' - -image: docker:latest - -stages: - - test - - deploy - -variables: - FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true" # see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/1736 - GITLABCI_VERSION: "22.04" - GITLABCI_IMAGE: "gitlabci" - CONTRIBUTOR_IMAGE: "domjudge/domjudge-contributor" - DOMJUDGE_VERSION: - value: "M.m.pp" - description: "The DOMjudge version, Change this variable to 7.3.3 to release the 7.3.3 dockers. The file should be available on the domjudge.org webserver." - DOMJUDGE_LATEST: - value: "true" - description: "Whether this is the latest release" - -# Docker Login steps -.release_template: &release_docker - services: - - docker:dind - -.registry_template: ®istry_gitlab - <<: *release_docker - stage: test - before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - -.registry_template: ®istry_dockerhub - <<: *release_docker - stage: deploy - before_script: - - docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_ACCESS_TOKEN - -.job_template: &job_check-pr - stage: test - services: - - docker:dind - except: - - main - -.matrix_contributor_template: &matrix_contributor_template - timeout: 90 minutes - parallel: - matrix: - - ARCH: [amd64] - -.job_template: &ci_template - script: - - cd docker-gitlabci - - > - if [[ -z $DOCKER_HUB_USERNAME ]]; then - BASE=$CI_REGISTRY_IMAGE/$GITLABCI_IMAGE - else - BASE="domjudge/"$GITLABCI_IMAGE - fi - - sh ./build.sh $GITLABCI_VERSION $BASE - - docker push $BASE:$GITLABCI_VERSION - -pr-ci: - <<: *registry_gitlab - <<: *ci_template - except: - - main - -release-ci: - <<: *registry_dockerhub - <<: *ci_template - only: - refs: - - main - -check-pr-contributor: - <<: *job_check-pr - <<: *matrix_contributor_template - script: - - cd docker-contributor - - docker pull $CONTRIBUTOR_IMAGE:${ARCH} || true - - docker build --build-arg ARCH=${ARCH}/ . - -release-contributor-arch: - <<: *registry_dockerhub - <<: *matrix_contributor_template - only: - refs: - - main - script: - - cd docker-contributor - - docker pull $CONTRIBUTOR_IMAGE:${ARCH} || true - - docker build -t $CONTRIBUTOR_IMAGE:${ARCH} --build-arg ARCH=${ARCH}/ . - - docker push $CONTRIBUTOR_IMAGE:${ARCH} - -release-contributor-latest: - <<: *registry_dockerhub - only: - refs: - - main - needs: - - release-contributor-arch - script: - - docker manifest create $CONTRIBUTOR_IMAGE:latest -a $CONTRIBUTOR_IMAGE:amd64 -a $CONTRIBUTOR_IMAGE:arm64v8 - - docker manifest push $CONTRIBUTOR_IMAGE:latest - -release-DOMjudge: - <<: *registry_dockerhub - when: manual - allow_failure: false - only: - - main - script: - - cd docker - - sh ./build.sh $DOMJUDGE_VERSION - - > - for IMG in domserver judgehost default-judgehost-chroot; do - docker push domjudge/$IMG:$DOMJUDGE_VERSION - if [ ${DOMJUDGE_LATEST} == "true" ]; then - docker tag domjudge/$IMG:$DOMJUDGE_VERSION domjudge/$IMG:latest - docker push domjudge/$IMG:latest - fi - done - -check-pr-DOMjudge: - <<: *job_check-pr - script: - - set -x - - HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" - - apk add jq curl - - TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[length-2].name') - - TAG=${TAG//\"} - - cd docker - - sh ./build.sh "$TAG" - - | - # check that there are no world-writable files - # (this is mainly a regression test for FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR) - # ignore symbolic links, because they always have mode "rwxrwxrwx" - # ignore directories with restricted deletion flag (e.g. /tmp), because they are fine - # ignore character devices (e.g. /chroot/domjudge/dev/* in image domjudge/judgehost are fine) - for IMG in domserver judgehost; do - files=$(docker run --rm --pull=never "domjudge/$IMG:$TAG" find / -xdev -perm -o+w ! -type l ! \( -type d -a -perm -+t \) ! -type c) - if [ -n "$files" ]; then - echo "error: image docker/$IMG contains world-writable files:" >&2 - printf "%s\n" "$files" >&2 - exit 1 - fi - done From da865fbb3f047e9222586a753aa0f8d3b06fe752 Mon Sep 17 00:00:00 2001 From: Michael <14887731+vmcj@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:52:42 +0100 Subject: [PATCH 13/61] Remove forgotten file --- .tarball-release.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .tarball-release.yml diff --git a/.tarball-release.yml b/.tarball-release.yml deleted file mode 100644 index 705627e9..00000000 --- a/.tarball-release.yml +++ /dev/null @@ -1,29 +0,0 @@ -# The jobs to do a manual DOMjudge release -createtar: - stage: test - when: manual - image: domjudge/gitlabci:2.1 - script: - - echo -e "\e[0Ksection_start:`date +%s`:buildrelease\r\e[0KBuild of release tarball" - - wget https://raw.githubusercontent.com/DOMjudge/domjudge-scripts/main/make_release.sh - - sh ./make_release.sh $DOMJUDGE_VERSION - - echo -e "\e[0Ksection_end:`date +%s`:buildrelease\r\e[0K" - - echo -e "\e[0Ksection_start:`date +%s`:aptpackages\r\e[0KInstall the expected packages for new release" - - apt update - - apt install -y acl zip unzip mariadb-server apache2 php php-fpm php-gd php-cli php-intl php-mbstring php-mysql php-curl php-json php-xml php-zip composer ntp - - apt install -y make pkg-config sudo debootstrap libcgroup-dev php-cli php-curl php-json php-xml php-zip lsof procps - - echo -e "\e[0Ksection_end:`date +%s`:aptpackages\r\e[0K" - - echo -e "\e[0Ksection_start:`date +%s`:testtar\r\e[0KTest the release tarball" - - tar zxvf domjudge-${DOMJUDGE_VERSION}.tar.gz - - cd domjudge-${DOMJUDGE_VERSION} - - ./configure - - make build - - make clean - - make all - - make clean - - make docs - - make clean - - make submitclient - - echo -e "\e[0Ksection_end:`date +%s`:testtar\r\e[0K" - - echo "See the new_release_howto.txt (L28/domjudge-scripts) for the next step." - From 88b96046e469cfe4d4d238b346eb6d2da10b2006 Mon Sep 17 00:00:00 2001 From: Michael <14887731+vmcj@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:49:45 +0100 Subject: [PATCH 14/61] Move contributor image creation to GitHub actions We could speed this up with creating the amd64 image in parallel but the bottleneck is always the arm64 as GitHub doesn't seem to have arm runners available yet. We now create a PR image which can be tested before in case this is needed. --- .../build-contributor-container-PR.yml | 42 +++++++++++++++++++ .../build-contributor-container-release.yml | 35 ++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 .github/workflows/build-contributor-container-PR.yml create mode 100644 .github/workflows/build-contributor-container-release.yml diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml new file mode 100644 index 00000000..d23b252b --- /dev/null +++ b/.github/workflows/build-contributor-container-PR.yml @@ -0,0 +1,42 @@ +name: 'Build Contributor container (PR)' +# For speedup to do this in parallel: https://docs.docker.com/build/ci/github-actions/multi-platform/ +# Loosely based on: https://docs.docker.com/build/ci/github-actions/push-multi-registries/ + +on: + push: + pull_request_target: + branches: + - main + +jobs: + pr-contributor: + if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'main' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ghcr.io/${{ github.repository }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: docker-contributor + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml new file mode 100644 index 00000000..f7d984e8 --- /dev/null +++ b/.github/workflows/build-contributor-container-release.yml @@ -0,0 +1,35 @@ +name: 'Build Contributor container (Release)' +# For speedup to do this in parallel: https://docs.docker.com/build/ci/github-actions/multi-platform/ +# Loosely based on: https://docs.docker.com/build/ci/github-actions/push-multi-registries/ + +on: + push: + pull_request_target: + branches: + - main + +jobs: + release-contributor: + if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'main' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: docker-contributor + platforms: linux/amd64,linux/arm64 + push: true + tags: domjudge/domjudge-contributor:latest From 8944e11a8ea1c38e7b7697895a9ab64d9d7848a8 Mon Sep 17 00:00:00 2001 From: Michael <14887731+vmcj@users.noreply.github.com> Date: Fri, 12 Jan 2024 15:56:30 +0100 Subject: [PATCH 15/61] Create action for docker build scripts The PRs for changes to those scripts will be stored in the registry of the user/organisation which forked or in our GitHub docker registry if this branch is under the domjudge organization. Here we always build against our latest version. The GitLab code had the option to not push the latest tag, for when we rebuild an older container, otherwise we always release against the overwritten value or if nothing was provided against the latest released tag (so which latest points to). The code for world readable files has been kept. Our build script is extended to now also have an option to push to another organization/namespace so we can push the image to the github container registry of the person doing the PR. As we don't do this often we explicit clean the github runner of older versions to make sure we always build against the latest image available of our dependencies and don't encounter the earlier builds if a PR is done more often (to fix something for example). The image can be locally tested by looking at the special tag based on the branchname/issue_number. --- .../workflows/build-domjudge-container-PR.yml | 83 +++++++++++++++++++ .../build-domjudge-container-release.yml | 65 +++++++++++++++ docker/build.sh | 28 ++++--- 3 files changed, 165 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/build-domjudge-container-PR.yml create mode 100644 .github/workflows/build-domjudge-container-release.yml diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml new file mode 100644 index 00000000..e84780f5 --- /dev/null +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -0,0 +1,83 @@ +name: 'Build domjudge container (PR)' + +on: + push: + pull_request_target: + branches: + - main + +env: + DOMJUDGE_VERSION: M.m.p + +jobs: + pr-domjudge: + if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'main' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - run: docker system prune -a -f + + - name: Get an unique tag for when people PR often + run: | + GHR=${{ github.ref }} + echo "PR_TAG=${GHR///}" >> $GITHUB_ENV + + - name: If needed overwrite the DOMJUDGE_VERSION for this run + run: | + if [ ${{ env.DOMJUDGE_VERSION }} != "M.m.p" ]; then + exit 0 + fi + sudo apt update; sudo apt install -y jq curl + set -x + HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" + TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[length-2].name') + DJ_TAG=${TAG//\"} + set +x + echo "DOMJUDGE_VERSION=$DJ_TAG" >> $GITHUB_ENV + + - name: Build the container + run: | + cd docker + set -x + sh ./build.sh "${{ env.DOMJUDGE_VERSION }}" ${{ github.actor }} + set +x + + - run: docker image list + + - name: Build and push + run: | + for IMG in domserver judgehost default-judgehost-chroot; do + IMAGE_NAME="${{ github.actor }}/$IMG:${{ env.DOMJUDGE_VERSION }}" + docker image tag "$IMAGE_NAME" ghcr.io/${{ github.actor }}/$IMG:${{ env.PR_TAG }} + docker image tag "$IMAGE_NAME" ${{ github.actor }}/$IMG:${{ env.PR_TAG }} + docker push ghcr.io/${{ github.actor }}/$IMG:${{ env.PR_TAG }} + done + + - name: Check for wrong permisions + run: | + docker image list + set -x + for IMG in domserver judgehost; do + files=$(docker run --rm --pull=never "${{ github.actor }}/$IMG:${{ env.PR_TAG }}" find / -xdev -perm -o+w ! -type l ! \( -type d -a -perm -+t \) ! -type c) + if [ -n "$files" ]; then + echo "error: image ${{ github.actor }}/$IMG:${{ env.PR_TAG }} contains world-writable files:" >&2 + printf "%s\n" "$files" >&2 + exit 1 + fi + done + diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml new file mode 100644 index 00000000..872b3d98 --- /dev/null +++ b/.github/workflows/build-domjudge-container-release.yml @@ -0,0 +1,65 @@ +name: 'Build domjudge container (Release)' + +on: + push: + pull_request_target: + branches: + - main + +env: + DOMJUDGE_VERSION: M.m.p + DOMJUDGE_LATEST: true + +jobs: + release-domjudge: + if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'main' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: If needed overwrite the DOMJUDGE_VERSION for this run + run: | + if [ ${{ env.DOMJUDGE_VERSION }} != "M.m.p" ]; then + exit 0 + fi + if [ ${{ env.DOMJUDGE_LATEST }} == "false" ]; then + echo "I don't know which version to pick!" + exit 1 + fi + apk add jq curl + set -x + HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" + TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[length-2].name') + DJ_TAG=${TAG//\"} + set +x + echo "DOMJUDGE_VERSION=$DJ_TAG" >> $GITHUB_ENV + + - name: Build the container + run: | + cd docker + set -x + sh ./build.sh "${{ env.DOMJUDGE_VERSION }}" + set +x + + - name: Build and push + run: | + for IMG in domserver judgehost default-judgehost-chroot; do + docker push domjudge/$IMG:${{ env.DOMJUDGE_VERSION }} + if [ ${{ env.DOMJUDGE_LATEST }} = "true" ]; then + docker tag domjudge/$IMG:${{ env.DOMJUDGE_VERSION }} domjudge/$IMG:latest + docker push domjudge/$IMG:latest + fi + done diff --git a/docker/build.sh b/docker/build.sh index 14c07432..525576ec 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -6,14 +6,20 @@ then export PS4='(${0}:${LINENO}): - [$?] $ ' fi -if [ "$#" -ne 1 ] +if [ "$#" -eq 0 ] || [ "$#" -gt 2 ] then - echo "Usage: $0 domjudge-version" + echo "Usage: $0 domjudge-version " echo " For example: $0 5.3.0" + echo " or: $0 5.3.0 otherNamespace" exit 1 fi VERSION="$1" +NAMESPACE="domjudge" +if [ -n "${2+x}" ] +then + NAMESPACE="$2" +fi URL=https://www.domjudge.org/releases/domjudge-${VERSION}.tar.gz FILE=domjudge.tar.gz @@ -29,22 +35,22 @@ fi echo "[ok] DOMjudge version ${VERSION} downloaded as domjudge.tar.gz"; echo echo "[..] Building Docker image for domserver..." -./build-domjudge.sh "domjudge/domserver:${VERSION}" +./build-domjudge.sh "${NAMESPACE}/domserver:${VERSION}" echo "[ok] Done building Docker image for domserver" echo "[..] Building Docker image for judgehost using intermediate build image..." -./build-judgehost.sh "domjudge/judgehost:${VERSION}" +./build-judgehost.sh "${NAMESPACE}/judgehost:${VERSION}" echo "[ok] Done building Docker image for judgehost" echo "[..] Building Docker image for judgehost chroot..." -docker build -t "domjudge/default-judgehost-chroot:${VERSION}" -f judgehost/Dockerfile.chroot . +docker build -t "${NAMESPACE}/default-judgehost-chroot:${VERSION}" -f judgehost/Dockerfile.chroot . echo "[ok] Done building Docker image for judgehost chroot" -echo "All done. Image domjudge/domserver:${VERSION} and domjudge/judgehost:${VERSION} created" +echo "All done. Image ${NAMESPACE}/domserver:${VERSION} and ${NAMESPACE}/judgehost:${VERSION} created" echo "If you are a DOMjudge maintainer with access to the domjudge organization on Docker Hub, you can now run the following command to push them to Docker Hub:" -echo "$ docker push domjudge/domserver:${VERSION} && docker push domjudge/judgehost:${VERSION} && docker push domjudge/default-judgehost-chroot:${VERSION}" +echo "$ docker push ${NAMESPACE}/domserver:${VERSION} && docker push ${NAMESPACE}/judgehost:${VERSION} && docker push $NAMESPACE}/default-judgehost-chroot:${VERSION}" echo "If this is the latest release, also run the following command:" -echo "$ docker tag domjudge/domserver:${VERSION} domjudge/domserver:latest && \ -docker tag domjudge/judgehost:${VERSION} domjudge/judgehost:latest && \ -docker tag domjudge/default-judgehost-chroot:${VERSION} domjudge/default-judgehost-chroot:latest && \ -docker push domjudge/domserver:latest && docker push domjudge/judgehost:latest && docker push domjudge/default-judgehost-chroot:latest" +echo "$ docker tag ${NAMESPACE}/domserver:${VERSION} ${NAMESPACE}/domserver:latest && \ +docker tag ${NAMESPACE}/judgehost:${VERSION} ${NAMESPACE}/judgehost:latest && \ +docker tag ${NAMESPACE}/default-judgehost-chroot:${VERSION} ${NAMESPACE}/default-judgehost-chroot:latest && \ +docker push ${NAMESPACE}/domserver:latest && docker push ${NAMESPACE}/judgehost:latest && docker push ${NAMESPACE}/default-judgehost-chroot:latest" From 0f87362b97a56de5fda6c0161b44bd7f4ea6da6f Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Mon, 15 Jan 2024 23:17:56 +0100 Subject: [PATCH 16/61] Repair the release The repo was not properly quoted and the github.ref has another format with direct push. The push should be done without the tag after the image. Push resulting image to our DOMjudge GitHub container registry Alternative is to push to the own doing the PR, but they would be able to push another image to have the risk that in theory we test another PR than was used in the code from the PR. It seems to try to push the latest tag so make which tag pushed explicit. --- .../workflows/build-contributor-container-PR.yml | 5 +++-- .../build-contributor-container-release.yml | 3 ++- .../workflows/build-domjudge-container-PR.yml | 16 ++++++++-------- .../build-domjudge-container-release.yml | 5 +++-- .github/workflows/build-gitlab-container-PR.yml | 3 ++- .../workflows/build-gitlab-container-release.yml | 4 +++- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index d23b252b..aa64cf52 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -10,7 +10,7 @@ on: jobs: pr-contributor: - if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'main' }} + if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -39,4 +39,5 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} + diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml index f7d984e8..d8976e16 100644 --- a/.github/workflows/build-contributor-container-release.yml +++ b/.github/workflows/build-contributor-container-release.yml @@ -10,7 +10,7 @@ on: jobs: release-contributor: - if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'main' }} + if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -33,3 +33,4 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: domjudge/domjudge-contributor:latest + diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index e84780f5..25b0fd10 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -11,7 +11,7 @@ env: jobs: pr-domjudge: - if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'main' }} + if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -35,7 +35,7 @@ jobs: - name: Get an unique tag for when people PR often run: | GHR=${{ github.ref }} - echo "PR_TAG=${GHR///}" >> $GITHUB_ENV + echo "PR_TAG=${GHR///}${{ github.actor }}" >> $GITHUB_ENV - name: If needed overwrite the DOMJUDGE_VERSION for this run run: | @@ -62,10 +62,10 @@ jobs: - name: Build and push run: | for IMG in domserver judgehost default-judgehost-chroot; do - IMAGE_NAME="${{ github.actor }}/$IMG:${{ env.DOMJUDGE_VERSION }}" - docker image tag "$IMAGE_NAME" ghcr.io/${{ github.actor }}/$IMG:${{ env.PR_TAG }} - docker image tag "$IMAGE_NAME" ${{ github.actor }}/$IMG:${{ env.PR_TAG }} - docker push ghcr.io/${{ github.actor }}/$IMG:${{ env.PR_TAG }} + IMAGE_NAME="${{ github.repository_owner }}/$IMG:${{ env.DOMJUDGE_VERSION }}" + docker image tag "$IMAGE_NAME" ghcr.io/${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} + docker image tag "$IMAGE_NAME" ${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} + docker push ghcr.io/${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} done - name: Check for wrong permisions @@ -73,9 +73,9 @@ jobs: docker image list set -x for IMG in domserver judgehost; do - files=$(docker run --rm --pull=never "${{ github.actor }}/$IMG:${{ env.PR_TAG }}" find / -xdev -perm -o+w ! -type l ! \( -type d -a -perm -+t \) ! -type c) + files=$(docker run --rm --pull=never "${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }}" find / -xdev -perm -o+w ! -type l ! \( -type d -a -perm -+t \) ! -type c) if [ -n "$files" ]; then - echo "error: image ${{ github.actor }}/$IMG:${{ env.PR_TAG }} contains world-writable files:" >&2 + echo "error: image ${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} contains world-writable files:" >&2 printf "%s\n" "$files" >&2 exit 1 fi diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 872b3d98..736d370a 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -12,7 +12,7 @@ env: jobs: release-domjudge: - if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'main' }} + if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -60,6 +60,7 @@ jobs: docker push domjudge/$IMG:${{ env.DOMJUDGE_VERSION }} if [ ${{ env.DOMJUDGE_LATEST }} = "true" ]; then docker tag domjudge/$IMG:${{ env.DOMJUDGE_VERSION }} domjudge/$IMG:latest - docker push domjudge/$IMG:latest + docker push domjudge/$IMG fi done + diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index 7ab62ea2..a0066610 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -8,7 +8,7 @@ on: jobs: pr-gitlab: - if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'main' }} + if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'refs/heads/main' }} name: PR GitLab image runs-on: ubuntu-latest permissions: @@ -40,3 +40,4 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml index 62f049db..ffbd0bed 100644 --- a/.github/workflows/build-gitlab-container-release.yml +++ b/.github/workflows/build-gitlab-container-release.yml @@ -4,9 +4,10 @@ on: push: branches: - main + jobs: build-gitlab: - if: ${{ github.repository == domjudge/domjudge-packaging && github.ref == main }} + if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main' }} name: Build GitLab image runs-on: ubuntu-latest permissions: @@ -38,3 +39,4 @@ jobs: push: true tags: "domjudge/gitlabci:22.04" labels: ${{ steps.meta.outputs.labels }} + From 41d0d0d06ce8c35c5550da297abf229bfe8b02f5 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 2 Feb 2024 23:29:27 +0100 Subject: [PATCH 17/61] Use the correct variables --- .github/workflows/build-contributor-container-release.yml | 4 ++-- .github/workflows/build-domjudge-container-release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml index d8976e16..ceac0f1e 100644 --- a/.github/workflows/build-contributor-container-release.yml +++ b/.github/workflows/build-contributor-container-release.yml @@ -23,8 +23,8 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v5 diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 736d370a..914e7d53 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -27,8 +27,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: If needed overwrite the DOMJUDGE_VERSION for this run run: | From 9c0247e9704f4a7c9f7bc4b24c497358f768fa39 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 2 Feb 2024 23:23:58 +0100 Subject: [PATCH 18/61] Keep consistent in naming --- .github/workflows/build-gitlab-container-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml index ffbd0bed..f36a3104 100644 --- a/.github/workflows/build-gitlab-container-release.yml +++ b/.github/workflows/build-gitlab-container-release.yml @@ -1,4 +1,4 @@ -name: 'Build GitLab CI container (Accepted)' +name: 'Build GitLab CI container (Release)' on: push: From da759fcc2add902ae229dd3293a1b3889171f682 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 2 Feb 2024 23:37:51 +0100 Subject: [PATCH 19/61] Clarify which registry is used - PRs/branches: ghcr - merged: DockerHub --- .github/workflows/build-contributor-container-PR.yml | 2 +- .github/workflows/build-contributor-container-release.yml | 1 - .github/workflows/build-domjudge-container-PR.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index aa64cf52..b3536fd2 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -19,7 +19,7 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub + - name: Login to GitHub container registry uses: docker/login-action@v3 with: registry: ghcr.io diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml index ceac0f1e..22835a88 100644 --- a/.github/workflows/build-contributor-container-release.yml +++ b/.github/workflows/build-contributor-container-release.yml @@ -22,7 +22,6 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - registry: ghcr.io username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 25b0fd10..573ffd66 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub + - name: Login to GitHub container registry uses: docker/login-action@v3 with: registry: ghcr.io From c25502159f376dca458e4dcdc9839dabe93d4b51 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 2 Feb 2024 22:17:25 +0100 Subject: [PATCH 20/61] Don't run this on branches in our own org We would trigger both on push and pull_request, skip the 2nd one. So we either run when this is a push in our organisation but not to main, OR if this is a pull_request from another organisation/user to domjudge_org. --- .github/workflows/build-contributor-container-PR.yml | 5 ++++- .github/workflows/build-domjudge-container-PR.yml | 5 ++++- .github/workflows/build-gitlab-container-PR.yml | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index b3536fd2..70dcab95 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -10,7 +10,10 @@ on: jobs: pr-contributor: - if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'refs/heads/main' }} + if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && + github.ref == 'refs/heads/main') && + (github.event_name == 'pull_request_target' || + github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 573ffd66..0c3f2146 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -11,7 +11,10 @@ env: jobs: pr-domjudge: - if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'refs/heads/main' }} + if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && + github.ref == 'refs/heads/main') && + (github.event_name == 'pull_request_target' || + github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index a0066610..e986eb66 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -8,7 +8,10 @@ on: jobs: pr-gitlab: - if: ${{ github.repository != 'domjudge/domjudge-packaging' || github.ref != 'refs/heads/main' }} + if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && + github.ref == 'refs/heads/main') && + (github.event_name == 'pull_request_target' || + github.event.pull_request.head.repo.full_name != github.repository) }} name: PR GitLab image runs-on: ubuntu-latest permissions: From 22ab8307977d590c98ec97a3f603f42231bb1c3d Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 2 Feb 2024 19:20:23 +0100 Subject: [PATCH 21/61] Don't build release image on PRs When we push to our own organization this would trigger. As we only care for the push target in our own repo this is much easier. If someone would for they would need to change this but that is up to them. --- .github/workflows/build-contributor-container-release.yml | 3 +-- .github/workflows/build-domjudge-container-release.yml | 3 +-- .github/workflows/build-gitlab-container-release.yml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml index 22835a88..5b1bce93 100644 --- a/.github/workflows/build-contributor-container-release.yml +++ b/.github/workflows/build-contributor-container-release.yml @@ -4,13 +4,12 @@ name: 'Build Contributor container (Release)' on: push: - pull_request_target: branches: - main jobs: release-contributor: - if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main' }} + if: ${{ github.repository == 'domjudge/domjudge-packaging' }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 914e7d53..45774912 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -2,7 +2,6 @@ name: 'Build domjudge container (Release)' on: push: - pull_request_target: branches: - main @@ -12,7 +11,7 @@ env: jobs: release-domjudge: - if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main' }} + if: ${{ github.repository == 'domjudge/domjudge-packaging' }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml index f36a3104..39191124 100644 --- a/.github/workflows/build-gitlab-container-release.yml +++ b/.github/workflows/build-gitlab-container-release.yml @@ -7,7 +7,7 @@ on: jobs: build-gitlab: - if: ${{ github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main' }} + if: ${{ github.repository == 'domjudge/domjudge-packaging' }} name: Build GitLab image runs-on: ubuntu-latest permissions: From c5148144d549b5571cfc862b2c2144bbb499dfbf Mon Sep 17 00:00:00 2001 From: MCJ Vasseur <14887731+vmcj@users.noreply.github.com> Date: Thu, 1 Feb 2024 21:36:14 +0100 Subject: [PATCH 22/61] Run {group,user}mod as root This broke when the container user changed from root -> domjudge and not all actions as root were prefixed with sudo. --- docker-contributor/scripts/start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-contributor/scripts/start.sh b/docker-contributor/scripts/start.sh index bf583e15..06e27f5b 100644 --- a/docker-contributor/scripts/start.sh +++ b/docker-contributor/scripts/start.sh @@ -10,8 +10,8 @@ function file_or_env { } echo "[..] Changing user/group ID" -groupmod -g "${GID}" domjudge -usermod -u "${UID}" domjudge +sudo groupmod -g "${GID}" domjudge +sudo usermod -u "${UID}" domjudge echo "[ok] User ID set to ${UID} and group ID set to ${GID}"; echo echo "[..] Setting timezone" From d54511ae3292d321c0798dd3c3fde4b8c9badb8c Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 3 Feb 2024 11:16:02 +0100 Subject: [PATCH 23/61] Use lowercase github organization name --- .github/workflows/build-domjudge-container-PR.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 0c3f2146..b2073fad 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -65,10 +65,10 @@ jobs: - name: Build and push run: | for IMG in domserver judgehost default-judgehost-chroot; do - IMAGE_NAME="${{ github.repository_owner }}/$IMG:${{ env.DOMJUDGE_VERSION }}" - docker image tag "$IMAGE_NAME" ghcr.io/${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} - docker image tag "$IMAGE_NAME" ${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} - docker push ghcr.io/${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} + IMAGE_NAME="${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.DOMJUDGE_VERSION }}" + docker image tag "$IMAGE_NAME" ghcr.io/${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} + docker image tag "$IMAGE_NAME" ${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} + docker push ghcr.io/${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} done - name: Check for wrong permisions From 3ee0b23c7c35b334882a91dee8945bf10b5bdb13 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 3 Feb 2024 11:23:00 +0100 Subject: [PATCH 24/61] GitHub CI does not runs on ubuntu instead of alpine --- .github/workflows/build-domjudge-container-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 45774912..b21c9fc8 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -38,7 +38,7 @@ jobs: echo "I don't know which version to pick!" exit 1 fi - apk add jq curl + sudo apt update; sudo apt install jq curl -y set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[length-2].name') From d5b552754ccbac6293f84fadc6dd752d9669a0f6 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 3 Feb 2024 11:45:38 +0100 Subject: [PATCH 25/61] Don't run the PR job on merge-queue The branchname would become something hard to read and the branchname chosen by the contributor should already have an image as we work from a PR. --- .github/workflows/build-contributor-container-PR.yml | 1 + .github/workflows/build-domjudge-container-PR.yml | 1 + .github/workflows/build-gitlab-container-PR.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index 70dcab95..314f8dd6 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -12,6 +12,7 @@ jobs: pr-contributor: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && + github.event.action != 'enqueued' && (github.event_name == 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index b2073fad..633e4ca2 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -13,6 +13,7 @@ jobs: pr-domjudge: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && + github.event.action != 'enqueued' && (github.event_name == 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index e986eb66..3477f700 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -10,6 +10,7 @@ jobs: pr-gitlab: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && + github.event.action != 'enqueued' && (github.event_name == 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) }} name: PR GitLab image From 427e0877441e2de5bbac0d1ce567506585ca03e4 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 3 Feb 2024 13:30:51 +0100 Subject: [PATCH 26/61] Detect mergequeue branch in another way Building the image for the readonly branch is not needed as we already know this should work in the PR, only if someone would force merging before the CI passes we would need this. --- .github/workflows/build-contributor-container-PR.yml | 2 +- .github/workflows/build-domjudge-container-PR.yml | 2 +- .github/workflows/build-gitlab-container-PR.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index 314f8dd6..a45a8251 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -12,7 +12,7 @@ jobs: pr-contributor: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && - github.event.action != 'enqueued' && + !contains(github.ref, 'gh-readonly-queue') && (github.event_name == 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 633e4ca2..727ae932 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -13,7 +13,7 @@ jobs: pr-domjudge: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && - github.event.action != 'enqueued' && + !contains(github.ref, 'gh-readonly-queue') && (github.event_name == 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index 3477f700..c9159f29 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -10,7 +10,7 @@ jobs: pr-gitlab: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && - github.event.action != 'enqueued' && + !contains(github.ref, 'gh-readonly-queue') && (github.event_name == 'pull_request_target' || github.event.pull_request.head.repo.full_name != github.repository) }} name: PR GitLab image From 62889e12b789d8b055ea2d15dc2f25e2fc03f6bc Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Tue, 6 Feb 2024 21:43:32 +0100 Subject: [PATCH 27/61] Add instructions for setting up Traefik in Docker Include an example which sets up a Traefik reverse proxy in Docker, including ACME for automated https certificate management. The deprecated and legacy `--link` flag of run commands is replaced with Docker networks in all run commands. --- docker/README.md | 60 ++++++++++++++++++++++++++++--- docker/docker-compose-traefik.yml | 28 +++++++++++++++ docker/docker-compose.yml | 17 +++++++++ 3 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 docker/docker-compose-traefik.yml diff --git a/docker/README.md b/docker/README.md index 2232f6fc..718c8942 100644 --- a/docker/README.md +++ b/docker/README.md @@ -33,12 +33,43 @@ These containers do not include MySQL / MariaDB; the [MariaDB](https://hub.docke These images are available on the [Docker Hub](https://hub.docker.com) as `domjudge/domserver` and `domjudge/judgehost`. +### DOMjudge network +For isolation, create a Docker network which will be used for the communication between DOMjudge-related containers. +```bash +docker network create dj +``` + +### Traefik container (Optional) +An easy way to get trusted certificates using ACME is running the Traefik reverse proxy in front of the DOMjudge stack. Create a Docker network which allows communication across compose stacks using: + +```bash +docker network create proxy_network +``` + +Now, you can deploy a Traefik reverse proxy. An example test deployment with **insecure** API access on port `8080` can be created using: + +```bash +docker run --name traefik --net proxy_network -p 80:80 -p 443:443 -p 8080:8080 -v /letsencrypt -v /var/run/docker.sock:/var/run/docker.sock:ro traefik:v2.10 \ + --api.insecure=true \ + --providers.docker=true \ + --providers.docker.exposedbydefault=false \ + --entrypoints.web.address=:80 \ + --entrypoints.websecure.address=:443 \ + --certificatesresolvers.myresolver.acme.email=your-email@example.com \ + --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web \ + --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json +``` + +This will start a Traefik container listening for http traffic on port `80`, https traffic on port `443`, and listening for Traefik dashboard traffic at port `8080`. +Traefik automatically listens to the Docker socket for running containers with labels defining reverse proxy routes. +It also defines an ACME resolver named `myresolver` which creates a Let's Encrypt ACME account using email address `your-email@example.com` used to create the TLS certificates for https traffic. + ### MariaDB container Before starting the containers, make sure you have a MySQL / MariaDB database somewhere. The easiest way to get one up and running is to use the [MariaDB](https://hub.docker.com/r/_/mariadb/) Docker container: ```bash -docker run -it --name dj-mariadb -e MYSQL_ROOT_PASSWORD=rootpw -e MYSQL_USER=domjudge -e MYSQL_PASSWORD=djpw -e MYSQL_DATABASE=domjudge -p 13306:3306 mariadb --max-connections=1000 +docker run -it --name dj-mariadb --net dj -e MYSQL_ROOT_PASSWORD=rootpw -e MYSQL_USER=domjudge -e MYSQL_PASSWORD=djpw -e MYSQL_DATABASE=domjudge -p 13306:3306 mariadb --max-connections=1000 ``` This will start a MariaDB container, set the root password to `rootpw`, create a MySQL user named `domjudge` with password `djpw` and create an empty database named `domjudge`. It will also expose the server on port `13306` on your local machine, so you can use your favorite MySQL GUI to connect to it. If you want to save the MySQL data after removing the container, please read the [MariaDB](https://hub.docker.com/r/_/mariadb/) Docker Hub page for more information. @@ -47,10 +78,10 @@ This will start a MariaDB container, set the root password to `rootpw`, create a Next, if you are on Linux make sure you have cgroups enabled. See the [DOMjudge documentation about setting up a judgehost](https://www.domjudge.org/docs/manual/master/install-judgehost.html#linux-control-groups) for information about how to do this. Docker on Windows and macOS actually use a small Linux VM which already has these options set. -Now you can run the domserver using the following command: +Without the optional Traefik reverse proxy, you can run the domserver using the following command: ```bash -docker run --link dj-mariadb:mariadb -it -e MYSQL_HOST=mariadb -e MYSQL_USER=domjudge -e MYSQL_DATABASE=domjudge -e MYSQL_PASSWORD=djpw -e MYSQL_ROOT_PASSWORD=rootpw -p 12345:80 --name domserver domjudge/domserver:latest +docker run -it --name domserver --net dj -e MYSQL_HOST=dj-mariadb -e MYSQL_USER=domjudge -e MYSQL_DATABASE=domjudge -e MYSQL_PASSWORD=djpw -e MYSQL_ROOT_PASSWORD=rootpw -p 12345:80 domjudge/domserver:latest ``` If you want a specific DOMjudge version instead of the latest, replace `latest` with the DOMjudge version (e.g. `5.3.0`). @@ -70,6 +101,26 @@ If you lose access to the admin user, see the [DOMjudge documentation on resetti Make a note of the password for the `judgehost` user, it will be used when the judgehost container is configured. The password can be changed from the web interface by editing the `judgehost` user. +For a deployment using the Traefik container with ACME on domain `domjudge.example.com`, run domserver using the following command: + +```bash +docker create -it --name domserver --net dj -e MYSQL_HOST=dj-mariadb -e MYSQL_USER=domjudge -e MYSQL_DATABASE=domjudge -e MYSQL_PASSWORD=djpw -e MYSQL_ROOT_PASSWORD=rootpw \ + -l "traefik.enable=true" \ + -l "traefik.http.services.domjudge.loadbalancer.server.port=80" \ + -l "traefik.http.routers.domjudge.rule=Host(\`domjudge.example.com\`)" \ + -l "traefik.http.routers.domjudge.entrypoints=web" \ + -l "traefik.http.routers.domjudgesecure.rule=Host(\`domjudge.example.com\`)" \ + -l "traefik.http.routers.domjudgesecure.entrypoints=websecure" \ + -l "traefik.http.routers.domjudgesecure.tls=true" \ + -l "traefik.http.routers.domjudgesecure.tls.certresolver=myresolver" \ + -l "traefik.docker.network=proxy_network" \ + domjudge/domserver:latest +docker network connect proxy_network domserver +docker start -a domserver +``` + +With DNS configured, you can now access the web interface on [http://domjudge.example.com/](http://domjudge.example.com/) or [https://domjudge.example.com/](https://domjudge.example.com/) and log in as admin. + #### Environment variables The following environment variables are supported by the `domserver` container: @@ -130,13 +181,14 @@ where `[service]` is one of `nginx` or `php`. #### Docker-compose See https://github.com/DOMjudge/domjudge-packaging/blob/main/docker/docker-compose.yml for a docker-compose example which automates the steps above. +When using the optional Traefik reverse proxy, deploy the stack defined in https://github.com/DOMjudge/domjudge-packaging/blob/main/docker/docker-compose-traefik.yml first and uncomment the relevant lines in the `docker-compose.yml` file. ### Judgehost container To run a single judgehost, run the following command: ```bash -docker run -it --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro --name judgehost-0 --link domserver:domserver --hostname judgedaemon-0 -e DAEMON_ID=0 domjudge/judgehost:latest +docker run -it --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro --name judgehost-0 --net dj --hostname judgedaemon-0 -e DAEMON_ID=0 domjudge/judgehost:latest ``` Again, replace `latest` with a specific version if desired. Make sure the version matches the version of the domserver. diff --git a/docker/docker-compose-traefik.yml b/docker/docker-compose-traefik.yml new file mode 100644 index 00000000..9327da47 --- /dev/null +++ b/docker/docker-compose-traefik.yml @@ -0,0 +1,28 @@ +version: '3' + +services: + traefik: + image: "traefik:v2.10" + container_name: "traefik" + command: + - --api.insecure=true + - --providers.docker=true + - --providers.docker.exposedbydefault=false + - --entrypoints.web.address=:80 + - --entrypoints.websecure.address=:443 + - --certificatesresolvers.myresolver.acme.email=your-email@example.com + - --certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web + - --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json + ports: + - "80:80" + - "443:443" + - "8080:8080" + volumes: + - /letsencrypt + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - proxy_network + +networks: + proxy_network: + external: true diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index db94888c..5cd06c9a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -25,3 +25,20 @@ services: - MYSQL_HOST=mariadb ports: - 12345:80 +# When using the Traefik reverse proxy container, uncomment the lines below. +# labels: +# - traefik.enable=true +# - traefik.http.services.domjudge.loadbalancer.server.port=80 +# - traefik.http.routers.domjudge.rule=Host(`domjudge.example.com`) +# - traefik.http.routers.domjudge.entrypoints=web +# - traefik.http.routers.domjudgesecure.rule=Host(`domjudge.example.com`) +# - traefik.http.routers.domjudgesecure.entrypoints=websecure +# - traefik.http.routers.domjudgesecure.tls=true +# - traefik.http.routers.domjudgesecure.tls.certresolver=myresolver +# - traefik.docker.network=proxy_network +# networks: +# - default +# - proxy_network +# networks: +# proxy_network: +# external: true From 43982ab87f25c623b2f8085e67bca48765279dc1 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:27:14 +0100 Subject: [PATCH 28/61] Always use the latest tag In the past we would search for the last pushed which would with semantic versioning be the same, this makes more sense IMHO. --- .github/workflows/build-domjudge-container-PR.yml | 2 +- .github/workflows/build-domjudge-container-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 727ae932..99a097d4 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -49,7 +49,7 @@ jobs: sudo apt update; sudo apt install -y jq curl set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" - TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[length-2].name') + TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | last.name') DJ_TAG=${TAG//\"} set +x echo "DOMJUDGE_VERSION=$DJ_TAG" >> $GITHUB_ENV diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index b21c9fc8..836412bc 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -41,7 +41,7 @@ jobs: sudo apt update; sudo apt install jq curl -y set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" - TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[length-2].name') + TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | last.name') DJ_TAG=${TAG//\"} set +x echo "DOMJUDGE_VERSION=$DJ_TAG" >> $GITHUB_ENV From c6dae90423bbb3d8d3318b72f9c4ba53ca3b384c Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:37:18 +0100 Subject: [PATCH 29/61] Find latest version tag which exists on domjudge.org We can't use latest as there is no release on domjudge.org with that name, but this makes atleast clear which index we try to get. This reverts commit 43982ab87f25c623b2f8085e67bca48765279dc1 and improves on it. --- .github/workflows/build-domjudge-container-PR.yml | 2 +- .github/workflows/build-domjudge-container-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 99a097d4..49141223 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -49,7 +49,7 @@ jobs: sudo apt update; sudo apt install -y jq curl set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" - TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | last.name') + TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[-2].name') DJ_TAG=${TAG//\"} set +x echo "DOMJUDGE_VERSION=$DJ_TAG" >> $GITHUB_ENV diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 836412bc..758a2395 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -41,7 +41,7 @@ jobs: sudo apt update; sudo apt install jq curl -y set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" - TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | last.name') + TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[-2].name') DJ_TAG=${TAG//\"} set +x echo "DOMJUDGE_VERSION=$DJ_TAG" >> $GITHUB_ENV From 34fe46ab6370e15a576b5348c285219f5aed1034 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 26 Apr 2024 08:01:32 +0200 Subject: [PATCH 30/61] Use latest ubuntu for maintainer container --- docker-contributor/Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index 9878831e..fca61946 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -1,5 +1,5 @@ ARG ARCH= -FROM "${ARCH}ubuntu:jammy" +FROM "${ARCH}ubuntu:noble" LABEL maintainer="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive \ @@ -14,7 +14,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ FPM_MAX_CHILDREN=40 \ DJ_SKIP_MAKE=0 \ DJ_DB_INSTALL_BARE=0 \ - PHPSUPPORTED="8.0 8.1 8.2 8.3" \ + PHPSUPPORTED="8.1 8.2 8.3" \ DEFAULTPHPVERSION="8.3" \ APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" @@ -23,9 +23,9 @@ RUN apt update \ && apt install --no-install-recommends --no-install-suggests -y \ dumb-init autoconf automake git acl \ gcc g++ make zip unzip mariadb-client \ - nginx php8.1 php8.1-cli php8.1-fpm php8.1-zip \ - php8.1-gd php8.1-curl php8.1-mysql php8.1-intl \ - php8.1-gmp php8.1-xml php8.1-mbstring php8.1-xdebug php8.1-pcov \ + nginx php8.3 php8.3-cli php8.3-fpm php8.3-zip \ + php8.3-gd php8.3-curl php8.3-mysql php8.3-intl \ + php8.3-gmp php8.3-xml php8.3-mbstring php8.3-xdebug php8.3-pcov \ bsdmainutils ntp lsof \ linuxdoc-tools linuxdoc-tools-text groff \ python3-sphinx python3-sphinx-rtd-theme python3-pip fontconfig python3-yaml \ @@ -44,6 +44,8 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log \ && mkdir -p /run/php +# Remove default ubuntu user +RUN userdel ubuntu && groupdel ubuntu || true # Set up users RUN groupadd -g $GID domjudge \ && useradd -u $UID -g $GID -m domjudge \ @@ -60,7 +62,7 @@ RUN apt update && \ # Install all supported PHP versions RUN add-apt-repository ppa:ondrej/php -y && apt update RUN for VERSION in $PHPSUPPORTED; do \ - if [ "${VERSION}" != "8.1" ]; then \ + if [ "${VERSION}" != "8.3" ]; then \ $APTINSTALL php${VERSION}; \ fi; \ done @@ -68,7 +70,7 @@ RUN PACKAGES=$(dpkg-query -f '${binary:Package}\n' -W|grep "^php.*-"); \ for PACKAGE in $PACKAGES; do \ PACKAGEALLVERSIONS="" && \ for VERSION in $PHPSUPPORTED; do \ - if [ "${VERSION}" != "8.1" ]; then \ + if [ "${VERSION}" != "8.3" ]; then \ PACKAGEALLVERSIONS="$PACKAGEALLVERSIONS php${VERSION}-${PACKAGE#php*-}"; \ fi; \ done; \ From 16c3a0697d1ea989ed2cfd4a4d88e7a99678fbf3 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 27 Apr 2024 10:14:08 +0200 Subject: [PATCH 31/61] Rebuild the contributor image every friday morning This was tried in the past but failed for some of our images, so this is the first check to make this work again. --- .github/workflows/build-contributor-container-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml index 5b1bce93..ad0c0f12 100644 --- a/.github/workflows/build-contributor-container-release.yml +++ b/.github/workflows/build-contributor-container-release.yml @@ -3,6 +3,8 @@ name: 'Build Contributor container (Release)' # Loosely based on: https://docs.docker.com/build/ci/github-actions/push-multi-registries/ on: + schedule: + - cron: "5 4 * * 5" push: branches: - main From 940500e42b75be7144fbde22c6cef98005ff3eac Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 28 Apr 2024 00:27:41 +0200 Subject: [PATCH 32/61] Fix build of gitlab container This is hard to test but uses the newer action which should have a newer version of node. Also added the automatic weekly rebuild of the image. --- .github/workflows/build-gitlab-container-PR.yml | 9 +++++---- .github/workflows/build-gitlab-container-release.yml | 12 +++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index c9159f29..b2e78778 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -19,13 +19,14 @@ jobs: packages: write contents: read steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 - name: Use the same PHP config as the contributor containers run: cp -r docker-{contributor,gitlabci}/php-config - name: Log in to GitHub Container registry (possibly at fork) - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} @@ -33,12 +34,12 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + uses: docker/metadata-action@v5 with: images: ghcr.io/${{ github.repository }} - name: Build and push Docker images - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + uses: docker/build-push-action@v5 with: context: "./docker-gitlabci" push: true diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml index 39191124..f927a461 100644 --- a/.github/workflows/build-gitlab-container-release.yml +++ b/.github/workflows/build-gitlab-container-release.yml @@ -1,6 +1,8 @@ name: 'Build GitLab CI container (Release)' on: + schedule: + - cron: "4 3 * * 5" push: branches: - main @@ -14,26 +16,26 @@ jobs: packages: write contents: read steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 - name: Use the same PHP config as the contributor containers run: cp -r docker-{contributor,gitlabci}/php-config - name: Log in to Docker Container registry - if: github.ref == 'refs/heads/main' - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + uses: docker/metadata-action@v5 with: images: domjudge/gitlabci - name: Build and push Docker images - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + uses: docker/build-push-action@v5 with: context: "./docker-gitlabci" push: true From 7c8d5b9e64536fc178ce4d1036b2d3fb1ea46f0a Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 28 Apr 2024 15:01:02 +0200 Subject: [PATCH 33/61] Upgrade gitlab image to 24.04 Also removed some unneeded packages as we don't run those tests anymore. We need some extra changes for the pa11y now as the chrome-browser gets installed in an unreachable location for the domjudge user. Also download the latest dictionary file now as we use the system codespell now. Composer is now again in the ubuntu repo's so we reverted to using that one. --- docker-gitlabci/Dockerfile | 44 ++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 81d43e70..58775246 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -1,44 +1,29 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive -ENV PHPSUPPORTED="7.2 7.3 8.0 8.1 8.2" +ENV PHPSUPPORTED="7.4 8.0 8.2 8.3" ENV APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" RUN apt update && apt install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ gcc g++ default-jre-headless default-jdk ghc fp-compiler libcgroup-dev \ - devscripts shellcheck nginx libboost-regex-dev \ - php8.1 php8.1-cli php8.1-gd php8.1-curl php8.1-mysql php8.1-gmp php8.1-zip php8.1-xml php8.1-mbstring php8.1-fpm php8.1-intl php8.1-pcov \ - # W3c test \ - httrack \ - # Visual regression browser \ - cutycapt xvfb openimageio-tools imagemagick \ + devscripts shellcheck nginx libboost-regex-dev composer codespell \ + php8.3 php8.3-cli php8.3-gd php8.3-curl php8.3-mysql php8.3-gmp php8.3-zip php8.3-xml php8.3-mbstring php8.3-fpm php8.3-intl php8.3-pcov \ # Submit client \ python3-requests python3-magic \ # Docs \ python3-sphinx python3-sphinx-rtd-theme rst2pdf fontconfig python3-yaml \ - texlive-latex-recommended texlive-latex-extra \ - texlive-fonts-recommended texlive-lang-european latexmk \ + texlive-latex-recommended texlive-latex-extra texlive-lang-european latexmk \ # Misc gitlab things \ mariadb-client curl build-essential packaging-dev \ git python3-pip moreutils w3m python3-yaml \ # Things we'd have in the chroot \ - ca-certificates default-jre-headless pypy locales software-properties-common \ + ca-certificates default-jre-headless pypy3 locales software-properties-common \ # W3c WCAG \ - npm libnss3 libcups2 libxss1 libasound2 libatk1.0-0 libatk-bridge2.0-0 libpangocairo-1.0-0 libgtk-3-0 \ - # Code coverage for unit test + npm \ + # Code coverage for unit test \ php-pear php-dev \ # Needed NPM packages \ - && npm install -g pa11y \ - # Needed python packages \ - && pip install codespell \ - && rm -rf /var/lib/apt/lists/* - -# Install composer -RUN apt update && \ - apt install --no-install-recommends --no-install-suggests -y ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \ - && mv /composer.phar /usr/local/bin/composer + && npm install -g pa11y puppeteer # Install needed global PHP modules RUN composer -n require justinrainbow/json-schema @@ -51,7 +36,8 @@ RUN add-apt-repository ppa:ondrej/php -y && apt update && \ for PACKAGE in $PACKAGES; do \ $APTINSTALL php${VERSION}-${PACKAGE#php*-}; \ done; \ - done && update-alternatives --set php /usr/bin/php8.1 + done && update-alternatives --set php /usr/bin/php8.3 \ + && rm -rf /var/lib/apt/lists/* # Put the gitlab user in sudo RUN echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers @@ -61,6 +47,14 @@ RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-0 RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-1 RUN groupadd domjudge-run +# Download chrome for the domjudge user +RUN su domjudge -c "npx puppeteer browsers install chrome" + +# Update dictionary to latest version +ENV GITHUBDICTIONARY="https://raw.githubusercontent.com/codespell-project/codespell/master/codespell_lib/data/dictionary.txt" +ENV CODESPELLDICTLOCATION="/usr/lib/python3/dist-packages/codespell_lib/data/dictionary.txt" +RUN wget $GITHUBDICTIONARY -O $CODESPELLDICTLOCATION + # Do some extra setup RUN mkdir -p /run/php \ && rm /etc/php/*/fpm/pool.d/www.conf From 4ffd7bc22996567716b9af24fb1105b5b9590639 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Mon, 29 Apr 2024 18:38:58 +0200 Subject: [PATCH 34/61] Add PHP version of 22.04 image We didn't install that version after the last upgrade. --- docker-gitlabci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 58775246..9904f833 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive -ENV PHPSUPPORTED="7.4 8.0 8.2 8.3" +ENV PHPSUPPORTED="7.4 8.0 8.1 8.2" ENV APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" RUN apt update && apt install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ From e1fcd7838ce69a325d4e4b28f12ac8cecd04fee3 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Mon, 29 Apr 2024 18:40:15 +0200 Subject: [PATCH 35/61] Rename image to new ubuntu version This means we published a gitlabci container 22.04 which was actually 24.04. --- .github/workflows/build-gitlab-container-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml index f927a461..27165364 100644 --- a/.github/workflows/build-gitlab-container-release.yml +++ b/.github/workflows/build-gitlab-container-release.yml @@ -39,6 +39,6 @@ jobs: with: context: "./docker-gitlabci" push: true - tags: "domjudge/gitlabci:22.04" + tags: "domjudge/gitlabci:24.04" labels: ${{ steps.meta.outputs.labels }} From b58c97b0849b8483bfa1bb44d80d59b92cb54847 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sat, 1 Jun 2024 11:56:23 +0200 Subject: [PATCH 36/61] Show "Run workflow" button --- .../build-contributor-container-release.yml | 1 + .../build-domjudge-container-release.yml | 16 ++++++++++++++++ .../workflows/build-gitlab-container-release.yml | 1 + 3 files changed, 18 insertions(+) diff --git a/.github/workflows/build-contributor-container-release.yml b/.github/workflows/build-contributor-container-release.yml index ad0c0f12..9d79cbf2 100644 --- a/.github/workflows/build-contributor-container-release.yml +++ b/.github/workflows/build-contributor-container-release.yml @@ -8,6 +8,7 @@ on: push: branches: - main + workflow_dispatch: jobs: release-contributor: diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 758a2395..c8ad82a4 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -4,6 +4,16 @@ on: push: branches: - main + workflow_dispatch: + inputs: + DOMJUDGE_VERSION: + description: 'Version of DOMjudge to build' + required: true + default: 'M.m.p' + DOMJUDGE_LATEST: + description: 'Overwrite the latest tag' + required: true + default: 'true' env: DOMJUDGE_VERSION: M.m.p @@ -29,6 +39,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: If manually dispatched use input variables + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + echo "DOMJUDGE_VERSION=${{ inputs.DOMJUDGE_VERSION }}" >> $GITHUB_ENV + echo "DOMJUDGE_LATEST=${{ inputs.DOMJUDGE_LATEST }}" >> $GITHUB_ENV + - name: If needed overwrite the DOMJUDGE_VERSION for this run run: | if [ ${{ env.DOMJUDGE_VERSION }} != "M.m.p" ]; then diff --git a/.github/workflows/build-gitlab-container-release.yml b/.github/workflows/build-gitlab-container-release.yml index 27165364..d9341af1 100644 --- a/.github/workflows/build-gitlab-container-release.yml +++ b/.github/workflows/build-gitlab-container-release.yml @@ -6,6 +6,7 @@ on: push: branches: - main + workflow_dispatch: jobs: build-gitlab: From 8ee337781d7450f73179cf11d870014ef4272f4d Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 2 Jun 2024 00:17:22 +0200 Subject: [PATCH 37/61] Annotate GitHub actions log The current build fails and the log is very hard to read --- .../workflows/build-domjudge-container-PR.yml | 2 + docker/build.sh | 50 +++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 49141223..1f607233 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -66,10 +66,12 @@ jobs: - name: Build and push run: | for IMG in domserver judgehost default-judgehost-chroot; do + echo "::group::$IMG" IMAGE_NAME="${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.DOMJUDGE_VERSION }}" docker image tag "$IMAGE_NAME" ghcr.io/${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} docker image tag "$IMAGE_NAME" ${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} docker push ghcr.io/${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} + echo "::endgroup::" done - name: Check for wrong permisions diff --git a/docker/build.sh b/docker/build.sh index 525576ec..1079e330 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,9 +1,43 @@ -#!/bin/sh -eu +#!/bin/sh -eux -if [ -n "${CI+}" ] +# Placeholders to annotate the Github actions logs +trace_on () { true; } +trace_off () { true; } +section_start () { true; } +section_end () { true; } + +if [ -n "${CI+x}" ] then + if [ -n "${GITHUB_ACTION+x}" ] set -x - export PS4='(${0}:${LINENO}): - [$?] $ ' + then + # Functions to annotate the Github actions logs + trace_on () { set -x; } + trace_off () { + { set +x; } 2>/dev/null + } + + section_start_internal () { + echo "::group::$1" + trace_on + } + + section_end_internal () { + echo "::endgroup::" + trace_on + } + + section_start () { + trace_off + section_start_internal "$@" + } + section_end () { + trace_off + section_end_internal + } + else + export PS4='(${0}:${LINENO}): - [$?] $ ' + fi fi if [ "$#" -eq 0 ] || [ "$#" -gt 2 ] @@ -24,6 +58,7 @@ fi URL=https://www.domjudge.org/releases/domjudge-${VERSION}.tar.gz FILE=domjudge.tar.gz +section_start "Download DOMjudge tarball" echo "[..] Downloading DOMjudge version ${VERSION}..." if ! wget --quiet "${URL}" -O ${FILE} @@ -33,19 +68,27 @@ then fi echo "[ok] DOMjudge version ${VERSION} downloaded as domjudge.tar.gz"; echo +section_end +section_start "Build domserver container" echo "[..] Building Docker image for domserver..." ./build-domjudge.sh "${NAMESPACE}/domserver:${VERSION}" echo "[ok] Done building Docker image for domserver" +section_end +section_start "Build judgehost container (with intermediate image)" echo "[..] Building Docker image for judgehost using intermediate build image..." ./build-judgehost.sh "${NAMESPACE}/judgehost:${VERSION}" echo "[ok] Done building Docker image for judgehost" +section_end +section_start "Build judgehost container (judging chroot)" echo "[..] Building Docker image for judgehost chroot..." docker build -t "${NAMESPACE}/default-judgehost-chroot:${VERSION}" -f judgehost/Dockerfile.chroot . echo "[ok] Done building Docker image for judgehost chroot" +section_end +section_start "Push instructions" echo "All done. Image ${NAMESPACE}/domserver:${VERSION} and ${NAMESPACE}/judgehost:${VERSION} created" echo "If you are a DOMjudge maintainer with access to the domjudge organization on Docker Hub, you can now run the following command to push them to Docker Hub:" echo "$ docker push ${NAMESPACE}/domserver:${VERSION} && docker push ${NAMESPACE}/judgehost:${VERSION} && docker push $NAMESPACE}/default-judgehost-chroot:${VERSION}" @@ -54,3 +97,4 @@ echo "$ docker tag ${NAMESPACE}/domserver:${VERSION} ${NAMESPACE}/domserver:late docker tag ${NAMESPACE}/judgehost:${VERSION} ${NAMESPACE}/judgehost:latest && \ docker tag ${NAMESPACE}/default-judgehost-chroot:${VERSION} ${NAMESPACE}/default-judgehost-chroot:latest && \ docker push ${NAMESPACE}/domserver:latest && docker push ${NAMESPACE}/judgehost:latest && docker push ${NAMESPACE}/default-judgehost-chroot:latest" +section_end From 1a89f09e7a1b8b289d0353ad790c5b7e0f501770 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 2 Jun 2024 14:28:01 +0200 Subject: [PATCH 38/61] GitHub recommends to not use `pull_request_target` See: https://runs-on.com/github-actions/pull-request-vs-pull-request-target/ --- .github/workflows/build-contributor-container-PR.yml | 4 ++-- .github/workflows/build-domjudge-container-PR.yml | 4 ++-- .github/workflows/build-gitlab-container-PR.yml | 4 ++-- .github/workflows/shellcheck.yml | 3 --- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index a45a8251..90fcc573 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -4,7 +4,7 @@ name: 'Build Contributor container (PR)' on: push: - pull_request_target: + pull_request: branches: - main @@ -13,7 +13,7 @@ jobs: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && !contains(github.ref, 'gh-readonly-queue') && - (github.event_name == 'pull_request_target' || + (github.event_name == 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 1f607233..80a170c5 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -2,7 +2,7 @@ name: 'Build domjudge container (PR)' on: push: - pull_request_target: + pull_request: branches: - main @@ -14,7 +14,7 @@ jobs: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && !contains(github.ref, 'gh-readonly-queue') && - (github.event_name == 'pull_request_target' || + (github.event_name == 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository) }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index b2e78778..0b02c3c2 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -2,7 +2,7 @@ name: 'Build GitLab CI container (PR)' on: push: - pull_request_target: + pull_request: branches: - main @@ -11,7 +11,7 @@ jobs: if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && github.ref == 'refs/heads/main') && !contains(github.ref, 'gh-readonly-queue') && - (github.event_name == 'pull_request_target' || + (github.event_name == 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository) }} name: PR GitLab image runs-on: ubuntu-latest diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 99cb18cd..c6fd65e9 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -7,9 +7,6 @@ on: pull_request: branches: - main - pull_request_target: - branches: - - main jobs: shellcheck: From a2a06fbedc72f11e4099c7375edff0f78fa09d4f Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 2 Jun 2024 16:13:48 +0200 Subject: [PATCH 39/61] Don't store the resulting image for PRs Moving this out of the security scope of the repository would make that we need to store this for the `github.author`. As we never used this before it's now taken out. --- .github/workflows/build-contributor-container-PR.yml | 2 +- .github/workflows/build-domjudge-container-PR.yml | 11 +++++------ .github/workflows/build-gitlab-container-PR.yml | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index 90fcc573..889466c1 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -41,7 +41,7 @@ jobs: with: context: docker-contributor platforms: linux/amd64,linux/arm64 - push: true + push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 80a170c5..ae4e8a4f 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -58,11 +58,9 @@ jobs: run: | cd docker set -x - sh ./build.sh "${{ env.DOMJUDGE_VERSION }}" ${{ github.actor }} + sh ./build.sh "${{ env.DOMJUDGE_VERSION }}" set +x - - run: docker image list - - name: Build and push run: | for IMG in domserver judgehost default-judgehost-chroot; do @@ -70,18 +68,19 @@ jobs: IMAGE_NAME="${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.DOMJUDGE_VERSION }}" docker image tag "$IMAGE_NAME" ghcr.io/${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} docker image tag "$IMAGE_NAME" ${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} - docker push ghcr.io/${GITHUB_REPOSITORY_OWNER@L}/$IMG:${{ env.PR_TAG }} echo "::endgroup::" done + - run: docker image list + - name: Check for wrong permisions run: | docker image list set -x for IMG in domserver judgehost; do - files=$(docker run --rm --pull=never "${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }}" find / -xdev -perm -o+w ! -type l ! \( -type d -a -perm -+t \) ! -type c) + files=$(docker run --rm --pull=never "domjudge/$IMG:${{ env.PR_TAG }}" find / -xdev -perm -o+w ! -type l ! \( -type d -a -perm -+t \) ! -type c) if [ -n "$files" ]; then - echo "error: image ${{ github.repository_owner }}/$IMG:${{ env.PR_TAG }} contains world-writable files:" >&2 + echo "error: image domjudge/$IMG:${{ env.PR_TAG }} contains world-writable files:" >&2 printf "%s\n" "$files" >&2 exit 1 fi diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index 0b02c3c2..ffdb67f9 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -42,7 +42,7 @@ jobs: uses: docker/build-push-action@v5 with: context: "./docker-gitlabci" - push: true + push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 58d6a0a84d075094cae86436af571309ad6a0d43 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 2 Jun 2024 11:25:32 +0200 Subject: [PATCH 40/61] Actually trigger the workflow for PRs The workflows did not always trigger for PRs versus normal pushes. --- .github/workflows/build-contributor-container-PR.yml | 12 +++++++----- .github/workflows/build-domjudge-container-PR.yml | 12 +++++++----- .github/workflows/build-gitlab-container-PR.yml | 12 +++++++----- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index 889466c1..df6e2466 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -10,11 +10,13 @@ on: jobs: pr-contributor: - if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && - github.ref == 'refs/heads/main') && - !contains(github.ref, 'gh-readonly-queue') && - (github.event_name == 'pull_request' || - github.event.pull_request.head.repo.full_name != github.repository) }} + # Stop processing if this is a merge-queue + # Stop processing if this is not against our repo + # Always run if this PR is not from our organization + # Or run if this PR is not `main` (So notQueue && ourRepo && (notPROurOrg || notMain)) + if : ${{ !contains(github.ref, 'gh-readonly-queue') && + github.repository == 'domjudge/domjudge-packaging' && + !(github.event.pull_request.head.repo.full_name == 'domjudge/domjudge-packaging' && github.ref == 'main') }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index ae4e8a4f..315916f8 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -11,11 +11,13 @@ env: jobs: pr-domjudge: - if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && - github.ref == 'refs/heads/main') && - !contains(github.ref, 'gh-readonly-queue') && - (github.event_name == 'pull_request' || - github.event.pull_request.head.repo.full_name != github.repository) }} + # Stop processing if this is a merge-queue + # Stop processing if this is not against our repo + # Always run if this PR is not from our organization + # Or run if this PR is not `main` (So notQueue && ourRepo && (notPROurOrg || notMain)) + if : ${{ !contains(github.ref, 'gh-readonly-queue') && + github.repository == 'domjudge/domjudge-packaging' && + !(github.event.pull_request.head.repo.full_name == 'domjudge/domjudge-packaging' && github.ref == 'main') }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index ffdb67f9..fb2aed72 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -8,11 +8,13 @@ on: jobs: pr-gitlab: - if: ${{ !(github.repository == 'domjudge/domjudge-packaging' && - github.ref == 'refs/heads/main') && - !contains(github.ref, 'gh-readonly-queue') && - (github.event_name == 'pull_request' || - github.event.pull_request.head.repo.full_name != github.repository) }} + # Stop processing if this is a merge-queue + # Stop processing if this is not against our repo + # Always run if this PR is not from our organization + # Or run if this PR is not `main` (So notQueue && ourRepo && (notPROurOrg || notMain)) + if : ${{ !contains(github.ref, 'gh-readonly-queue') && + github.repository == 'domjudge/domjudge-packaging' && + !(github.event.pull_request.head.repo.full_name == 'domjudge/domjudge-packaging' && github.ref == 'main') }} name: PR GitLab image runs-on: ubuntu-latest permissions: From 49329b52b3026f188db6d8ec2ed0a8b69adabab7 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 2 Jun 2024 19:59:05 +0200 Subject: [PATCH 41/61] Only run when a file has changed --- .github/workflows/build-contributor-container-PR.yml | 6 ++++++ .github/workflows/build-domjudge-container-PR.yml | 6 ++++++ .github/workflows/build-gitlab-container-PR.yml | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index df6e2466..ecd2e2c5 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -4,9 +4,15 @@ name: 'Build Contributor container (PR)' on: push: + paths: + - docker-contributor + - .github/workflows/build-contributor-container-PR.yml pull_request: branches: - main + paths: + - docker-contributor + - .github/workflows/build-contributor-container-PR.yml jobs: pr-contributor: diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 315916f8..b40a3779 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -2,9 +2,15 @@ name: 'Build domjudge container (PR)' on: push: + paths: + - .github/workflows/build-domjudge-container-PR.yml + - docker pull_request: branches: - main + paths: + - .github/workflows/build-domjudge-container-PR.yml + - docker env: DOMJUDGE_VERSION: M.m.p diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index fb2aed72..776830c1 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -2,9 +2,17 @@ name: 'Build GitLab CI container (PR)' on: push: + paths: + - docker-contributor/php-config + - docker-gitlabci + - .github/workflows/build-gitlab-container-PR.yml pull_request: branches: - main + paths: + - docker-contributor/php-config + - docker-gitlabci + - .github/workflows/build-gitlab-container-PR.yml jobs: pr-gitlab: From 5c48773c1554b377e0312cb32b15dece2fb55226 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Mon, 3 Jun 2024 19:54:48 +0200 Subject: [PATCH 42/61] Install pkg-config to detect systemd files locations --- docker/domserver/Dockerfile | 2 +- docker/judgehost/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/domserver/Dockerfile b/docker/domserver/Dockerfile index 7c92664d..5272ffda 100644 --- a/docker/domserver/Dockerfile +++ b/docker/domserver/Dockerfile @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Install required packages for build of domserver RUN apt update \ && apt install --no-install-recommends --no-install-suggests -y \ - autoconf automake git \ + autoconf automake git pkg-config \ gcc g++ make acl zip unzip \ php-cli php-zip \ php-gd php-curl php-mysql php-json php-intl \ diff --git a/docker/judgehost/Dockerfile b/docker/judgehost/Dockerfile index 0f06dcd9..18fcd179 100644 --- a/docker/judgehost/Dockerfile +++ b/docker/judgehost/Dockerfile @@ -13,7 +13,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ # Install required packages for running of judgehost RUN apt update \ && apt install --no-install-recommends --no-install-suggests -y \ - dumb-init \ + dumb-init pkg-config \ acl lsof zip unzip supervisor sudo procps libcgroup2 \ php-cli php-zip php-gd php-curl php-mysql php-json \ php-gmp php-xml php-mbstring python3 \ From 57bbd8f7779ff900f290438bdee2e64e48268042 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Mon, 3 Jun 2024 22:37:36 +0200 Subject: [PATCH 43/61] Also add pkg-config for the intermediate build --- docker/judgehost/Dockerfile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/judgehost/Dockerfile.build b/docker/judgehost/Dockerfile.build index 9b373553..08c4c0eb 100644 --- a/docker/judgehost/Dockerfile.build +++ b/docker/judgehost/Dockerfile.build @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Install required packages for build of judgehost RUN apt update \ && apt install --no-install-recommends --no-install-suggests -y \ - autoconf automake git \ + autoconf automake git pkg-config \ gcc g++ make zip unzip \ php-cli php-zip lsb-release debootstrap \ php-gd php-curl php-mysql php-json \ From 2608426f9262cde350fb8b73edfef5f4ff25dfa3 Mon Sep 17 00:00:00 2001 From: MCJ Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sun, 7 Jul 2024 10:11:47 +0200 Subject: [PATCH 44/61] Don't mount the cgroup as read-only Fixes: https://github.com/DOMjudge/domjudge-packaging/issues/193 I wonder why we never had this in the past as we do intent to create the domjudge cgroup in that directory. --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 718c8942..828999f4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -188,7 +188,7 @@ When using the optional Traefik reverse proxy, deploy the stack defined in https To run a single judgehost, run the following command: ```bash -docker run -it --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro --name judgehost-0 --net dj --hostname judgedaemon-0 -e DAEMON_ID=0 domjudge/judgehost:latest +docker run -it --privileged -v /sys/fs/cgroup:/sys/fs/cgroup --name judgehost-0 --net dj --hostname judgedaemon-0 -e DAEMON_ID=0 domjudge/judgehost:latest ``` Again, replace `latest` with a specific version if desired. Make sure the version matches the version of the domserver. @@ -242,7 +242,7 @@ echo 127.0.0.1 $(hostname) | sudo tee -a /etc/hosts ################################################### # Fill in these (secret) variables yourself!! -sudo docker run -d --restart=on-failure --network host --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro --name judgehost -e DOMSERVER_BASEURL=your_baseurl -e JUDGEDAEMON_USERNAME=your_username -e JUDGEDAEMON_PASSWORD=your_password domjudge/judgehost:7.0.3 +sudo docker run -d --restart=on-failure --network host --privileged -v /sys/fs/cgroup:/sys/fs/cgroup --name judgehost -e DOMSERVER_BASEURL=your_baseurl -e JUDGEDAEMON_USERNAME=your_username -e JUDGEDAEMON_PASSWORD=your_password domjudge/judgehost:7.0.3 ################################################### # Enable cgroup functionality that judgehost needs, this requires a reboot From 7d40d40585c7c8d64f8024bff971fe9a9de43671 Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Fri, 26 Apr 2024 16:09:21 +0000 Subject: [PATCH 45/61] Switch from 'apt' back to 'apt-get' The newer 'apt' command is recommended for interactive use, but not for scripts. Running 'apt' non-interactively produces the following message: WARNING: apt does not have a stable CLI interface. Use with caution in scripts. --- .github/workflows/build-domjudge-container-PR.yml | 2 +- .../workflows/build-domjudge-container-release.yml | 2 +- docker-contributor/Dockerfile | 12 ++++++------ docker-gitlabci/Dockerfile | 6 +++--- docker/domserver/Dockerfile | 8 ++++---- docker/judgehost/Dockerfile | 4 ++-- docker/judgehost/Dockerfile.build | 4 ++-- docker/judgehost/Dockerfile.chroot | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index b40a3779..c64b80ed 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -54,7 +54,7 @@ jobs: if [ ${{ env.DOMJUDGE_VERSION }} != "M.m.p" ]; then exit 0 fi - sudo apt update; sudo apt install -y jq curl + sudo apt-get update; sudo apt-get install -y jq curl set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[-2].name') diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index c8ad82a4..27f70310 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -54,7 +54,7 @@ jobs: echo "I don't know which version to pick!" exit 1 fi - sudo apt update; sudo apt install jq curl -y + sudo apt-get update; sudo apt-get install jq curl -y set -x HUBURL="https://registry.hub.docker.com/v2/repositories/domjudge/domserver/tags" TAG=$(curl $HUBURL|jq '.results | sort_by(.name) | .[-2].name') diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index fca61946..28659894 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -16,11 +16,11 @@ ENV DEBIAN_FRONTEND=noninteractive \ DJ_DB_INSTALL_BARE=0 \ PHPSUPPORTED="8.1 8.2 8.3" \ DEFAULTPHPVERSION="8.3" \ - APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" + APTINSTALL="apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" # Install required packages and clean up afterwards to make this image layer smaller -RUN apt update \ - && apt install --no-install-recommends --no-install-suggests -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ dumb-init autoconf automake git acl \ gcc g++ make zip unzip mariadb-client \ nginx php8.3 php8.3-cli php8.3-fpm php8.3-zip \ @@ -53,14 +53,14 @@ RUN groupadd -g $GID domjudge \ && for id in $(seq 0 4); do useradd -d /nonexistent -g nogroup -s /bin/false "domjudge-run-$id"; done # Install composer -RUN apt update && \ - apt install --no-install-recommends --no-install-suggests -y ca-certificates \ +RUN apt-get update && \ + apt-get install --no-install-recommends --no-install-suggests -y ca-certificates \ && rm -rf /var/lib/apt/lists/* \ && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \ && mv /composer.phar /usr/local/bin/composer # Install all supported PHP versions -RUN add-apt-repository ppa:ondrej/php -y && apt update +RUN add-apt-repository ppa:ondrej/php -y && apt-get update RUN for VERSION in $PHPSUPPORTED; do \ if [ "${VERSION}" != "8.3" ]; then \ $APTINSTALL php${VERSION}; \ diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 9904f833..08c1edce 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -1,8 +1,8 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive ENV PHPSUPPORTED="7.4 8.0 8.1 8.2" -ENV APTINSTALL="apt install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" -RUN apt update && apt install -y \ +ENV APTINSTALL="apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" +RUN apt-get update && apt-get install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ gcc g++ default-jre-headless default-jdk ghc fp-compiler libcgroup-dev \ @@ -29,7 +29,7 @@ RUN apt update && apt install -y \ RUN composer -n require justinrainbow/json-schema # Install other PHP versions -RUN add-apt-repository ppa:ondrej/php -y && apt update && \ +RUN add-apt-repository ppa:ondrej/php -y && apt-get update && \ PACKAGES=$(dpkg-query -f '${binary:Package}\n' -W|grep "^php.*-") && \ for VERSION in $PHPSUPPORTED; do \ $APTINSTALL php${VERSION} && \ diff --git a/docker/domserver/Dockerfile b/docker/domserver/Dockerfile index 5272ffda..22051c1e 100644 --- a/docker/domserver/Dockerfile +++ b/docker/domserver/Dockerfile @@ -4,8 +4,8 @@ LABEL org.opencontainers.image.authors="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive # Install required packages for build of domserver -RUN apt update \ - && apt install --no-install-recommends --no-install-suggests -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf automake git pkg-config \ gcc g++ make acl zip unzip \ php-cli php-zip \ @@ -52,8 +52,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ DJ_DB_INSTALL_BARE=0 # Install required packages for running of domserver -RUN apt update \ - && apt install --no-install-recommends --no-install-suggests -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ acl curl zip unzip supervisor mariadb-client apache2-utils \ nginx php-cli php-fpm php-zip \ php-gd php-curl php-mysql php-json php-intl \ diff --git a/docker/judgehost/Dockerfile b/docker/judgehost/Dockerfile index 18fcd179..ea5c22a4 100644 --- a/docker/judgehost/Dockerfile +++ b/docker/judgehost/Dockerfile @@ -11,8 +11,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN_USER_UID_GID=62860 # Install required packages for running of judgehost -RUN apt update \ - && apt install --no-install-recommends --no-install-suggests -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ dumb-init pkg-config \ acl lsof zip unzip supervisor sudo procps libcgroup2 \ php-cli php-zip php-gd php-curl php-mysql php-json \ diff --git a/docker/judgehost/Dockerfile.build b/docker/judgehost/Dockerfile.build index 08c4c0eb..716dda59 100644 --- a/docker/judgehost/Dockerfile.build +++ b/docker/judgehost/Dockerfile.build @@ -4,8 +4,8 @@ LABEL org.opencontainers.image.authors="DOMjudge team " ENV DEBIAN_FRONTEND=noninteractive # Install required packages for build of judgehost -RUN apt update \ - && apt install --no-install-recommends --no-install-suggests -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ autoconf automake git pkg-config \ gcc g++ make zip unzip \ php-cli php-zip lsb-release debootstrap \ diff --git a/docker/judgehost/Dockerfile.chroot b/docker/judgehost/Dockerfile.chroot index d533a5cc..89e9ade8 100644 --- a/docker/judgehost/Dockerfile.chroot +++ b/docker/judgehost/Dockerfile.chroot @@ -1,6 +1,6 @@ FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && apt -y install \ +RUN apt-get update && apt-get -y install \ ca-certificates default-jre-headless pypy3 locales \ && rm -rf /var/lib/apt/lists/* From 986b352957ac4d983d96e0a90582e2e5a13b8b8b Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Fri, 26 Apr 2024 16:40:32 +0000 Subject: [PATCH 46/61] Remove redundant 'apt-get install ca-certificates' That package is already installed earlier in the same file. --- docker-contributor/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index 28659894..d3119785 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -53,10 +53,7 @@ RUN groupadd -g $GID domjudge \ && for id in $(seq 0 4); do useradd -d /nonexistent -g nogroup -s /bin/false "domjudge-run-$id"; done # Install composer -RUN apt-get update && \ - apt-get install --no-install-recommends --no-install-suggests -y ca-certificates \ - && rm -rf /var/lib/apt/lists/* \ - && php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \ +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php \ && mv /composer.phar /usr/local/bin/composer # Install all supported PHP versions From d60e5e5c6b4f56e81de35e8d6f67cdc8e5c0f7da Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Tue, 9 Jul 2024 12:14:29 +0000 Subject: [PATCH 47/61] Restore `rm /var/lib/apt/lists` in docker-gitlabci/Dockerfile It was removed as part of commit 7c8d5b9e6 (Upgrade gitlab image to 24.04, 2024-04-28), but it should be kept to reduce the size of the image (if the lists aren't removed in the same RUN directive as `apt-get update` then they will be stored in the layer and make the image larger, even if a later RUN directive removes the lists). --- docker-gitlabci/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 08c1edce..3e95d80c 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -23,7 +23,8 @@ RUN apt-get update && apt-get install -y \ # Code coverage for unit test \ php-pear php-dev \ # Needed NPM packages \ - && npm install -g pa11y puppeteer + && npm install -g pa11y puppeteer \ + && rm -rf /var/lib/apt/lists/* # Install needed global PHP modules RUN composer -n require justinrainbow/json-schema From d1da7b7ff01c8545f79bece6cf65cb4224c71b2c Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Fri, 26 Apr 2024 16:40:44 +0000 Subject: [PATCH 48/61] Remove apt lists after 'apt-get update' in docker-contributor/Dockerfile We already do this in all other places. It reduces the size of the image. (Note that the 'rm' command has to be done in the same RUN instruction as 'apt-get update', otherwise the lists will still be stored in the intermediate layers. So some of the RUN instructions had to be merged.) --- docker-contributor/Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index d3119785..d99270cd 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -57,13 +57,13 @@ RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && && mv /composer.phar /usr/local/bin/composer # Install all supported PHP versions -RUN add-apt-repository ppa:ondrej/php -y && apt-get update -RUN for VERSION in $PHPSUPPORTED; do \ +RUN add-apt-repository ppa:ondrej/php -y && apt-get update && \ + for VERSION in $PHPSUPPORTED; do \ if [ "${VERSION}" != "8.3" ]; then \ $APTINSTALL php${VERSION}; \ fi; \ - done -RUN PACKAGES=$(dpkg-query -f '${binary:Package}\n' -W|grep "^php.*-"); \ + done && \ + PACKAGES=$(dpkg-query -f '${binary:Package}\n' -W|grep "^php.*-") && \ for PACKAGE in $PACKAGES; do \ PACKAGEALLVERSIONS="" && \ for VERSION in $PHPSUPPORTED; do \ @@ -72,8 +72,9 @@ RUN PACKAGES=$(dpkg-query -f '${binary:Package}\n' -W|grep "^php.*-"); \ fi; \ done; \ $APTINSTALL $PACKAGEALLVERSIONS; \ - done -RUN update-alternatives --set php /usr/bin/php${DEFAULTPHPVERSION} + done && \ + update-alternatives --set php /usr/bin/php${DEFAULTPHPVERSION} && \ + rm -rf /var/lib/apt/lists/* # Set up alternatives for PHP-FPM RUN for VERSION in $PHPSUPPORTED; do \ @@ -88,7 +89,7 @@ COPY ["php-config", "/php-config"] RUN for VERSION in $PHPSUPPORTED; do \ cp -Rf /php-config/* /etc/php/${VERSION}/cli/conf.d; \ cp -Rf /php-config/* /etc/php/${VERSION}/fpm/conf.d; \ - done; \ + done; \ rm -Rf /php-config # Disable Xdebug by default From a2b082448cad974525d16fea910e8b9f64ac8a17 Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Tue, 9 Jul 2024 16:54:57 +0000 Subject: [PATCH 49/61] Clean up docker/build.sh - Move usage check to the top, to serve as documentation for people who read the script. - Remove `-x` from shebang line (#!), to only enable tracing in CI. - Fix `set -x` call (previously it was inside the `if` condition, which is wrong). - Remove PS4 variable (which adds info to the trace) because sh does not support LINENO, and the rest of the info is not that useful. - Inline `trace_off` into section_start and section_end to produce less noise in the trace. (The "section_start" lines will still be displayed though. Hiding them is tricky; see DOMjudge/domjudge/gitlab/integration.sh for an example, but note that it requires bash for `shopt -s expand_aliases`.) - Make the no-op placeholders for section_start and section_end produce less noise in the trace. - Redirect stderr to stdout as a workaround for a GitHub Actions issue that causes them to appear out-of-order. - Simplify initialisation of NAMESPACE variable to make the trace look nicer. - Put the variable assignments in a log group to make it look nicer. - Fix the invocation of build.sh in the build-domjudge-container-* workflows to use `./build.sh` rather than `sh ./build.sh` so that the options in the shebang line will be respected. Also remove unnecessary calls to `set -x` in the workflows. --- .../workflows/build-domjudge-container-PR.yml | 4 +- .../build-domjudge-container-release.yml | 4 +- docker/build.sh | 73 ++++++++----------- 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index c64b80ed..0f5403d7 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -65,9 +65,7 @@ jobs: - name: Build the container run: | cd docker - set -x - sh ./build.sh "${{ env.DOMJUDGE_VERSION }}" - set +x + ./build.sh "${{ env.DOMJUDGE_VERSION }}" - name: Build and push run: | diff --git a/.github/workflows/build-domjudge-container-release.yml b/.github/workflows/build-domjudge-container-release.yml index 27f70310..9ad8ff42 100644 --- a/.github/workflows/build-domjudge-container-release.yml +++ b/.github/workflows/build-domjudge-container-release.yml @@ -65,9 +65,7 @@ jobs: - name: Build the container run: | cd docker - set -x - sh ./build.sh "${{ env.DOMJUDGE_VERSION }}" - set +x + ./build.sh "${{ env.DOMJUDGE_VERSION }}" - name: Build and push run: | diff --git a/docker/build.sh b/docker/build.sh index 1079e330..c4ba8d12 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,72 +1,59 @@ -#!/bin/sh -eux +#!/bin/sh -eu -# Placeholders to annotate the Github actions logs -trace_on () { true; } -trace_off () { true; } -section_start () { true; } -section_end () { true; } +if [ "$#" -eq 0 ] || [ "$#" -gt 2 ] +then + echo "Usage: $0 domjudge-version " + echo " For example: $0 5.3.0" + echo " or: $0 5.3.0 otherNamespace" + exit 1 +fi + +# Placeholders for grouping log lines +# (the body is a nested function declaration so it won't appear in the trace when using `set -x`) +section_start() { _() { :; }; } +section_end() { _() { :; }; } if [ -n "${CI+x}" ] then if [ -n "${GITHUB_ACTION+x}" ] - set -x then - # Functions to annotate the Github actions logs - trace_on () { set -x; } - trace_off () { - { set +x; } 2>/dev/null - } - - section_start_internal () { + # Functions for grouping log lines on GitHub Actions + trace_on() { set -x; } + # trace_off is manually inlined so it won't appear in the trace + section_start() { + { set +x; } 2>/dev/null # trace_off echo "::group::$1" trace_on } - - section_end_internal () { + section_end() { + { set +x; } 2>/dev/null # trace_off echo "::endgroup::" trace_on } - - section_start () { - trace_off - section_start_internal "$@" - } - section_end () { - trace_off - section_end_internal - } - else - export PS4='(${0}:${LINENO}): - [$?] $ ' + # Redirect stderr to stdout as a workaround so they won't be out-of-order; see + # https://github.com/orgs/community/discussions/116552 + # https://web.archive.org/web/20220430214837/https://github.community/t/stdout-stderr-output-not-in-correct-order-in-logs/16335 + # (GitHub Actions displays stderr in the same style as stdout anyway, so + # there is no harm in us merging them.) + exec 2>&1 fi + set -x fi -if [ "$#" -eq 0 ] || [ "$#" -gt 2 ] -then - echo "Usage: $0 domjudge-version " - echo " For example: $0 5.3.0" - echo " or: $0 5.3.0 otherNamespace" - exit 1 -fi - +section_start "Variables" VERSION="$1" -NAMESPACE="domjudge" -if [ -n "${2+x}" ] -then - NAMESPACE="$2" -fi - +NAMESPACE="${2-domjudge}" URL=https://www.domjudge.org/releases/domjudge-${VERSION}.tar.gz FILE=domjudge.tar.gz +section_end section_start "Download DOMjudge tarball" echo "[..] Downloading DOMjudge version ${VERSION}..." - if ! wget --quiet "${URL}" -O ${FILE} then echo "[!!] DOMjudge version ${VERSION} file not found on https://www.domjudge.org/releases" exit 1 fi - echo "[ok] DOMjudge version ${VERSION} downloaded as domjudge.tar.gz"; echo section_end From b2ccd49efeae06ef42702309a30a3e3f5fa580ea Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Sun, 21 Jul 2024 11:51:36 +0000 Subject: [PATCH 50/61] Remove obsolete reference to `--link` from docker/README.md Commit 62889e12b7 (Add instructions for setting up Traefik in Docker, 2024-02-06) switched from the legacy `--link` option to a user-defined bridge network (`--net`). --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 828999f4..9b46f867 100644 --- a/docker/README.md +++ b/docker/README.md @@ -126,7 +126,7 @@ With DNS configured, you can now access the web interface on [http://domjudge.ex The following environment variables are supported by the `domserver` container: * `CONTAINER_TIMEZONE` (defaults to `Europe/Amsterdam`): allows you to change the timezone used inside the container. -* `MYSQL_HOST` (defaults to `mariadb`): set the host to connect to for MySQL. Can be hostname or IP. Docker will add hostnames for any containers you `--link`, so in the example above, the MariaDB container will be available under the hostname `mariadb`. +* `MYSQL_HOST` (defaults to `mariadb`): set the host to connect to for MySQL. Can be hostname or IP. * `MYSQL_USER` (defaults to `domjudge`): set the user to use for connecting to MySQL. * `MYSQL_PASSWORD` (defaults to `domjudge`): set the password to use for connecting to MySQL. * `MYSQL_ROOT_PASSWORD` (defaults to `domjudge`): set the root password to use for connecting to MySQL. From 3a56ee9f3c1f1282f6050afe205c38afbe7a5e29 Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Thu, 23 Jul 2020 21:45:22 +0000 Subject: [PATCH 51/61] Abort on errors in chroot-and-tar.sh when building judgehost docker image --- docker/judgehost/chroot-and-tar.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/judgehost/chroot-and-tar.sh b/docker/judgehost/chroot-and-tar.sh index 15a00394..56da9a90 100755 --- a/docker/judgehost/chroot-and-tar.sh +++ b/docker/judgehost/chroot-and-tar.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -euo pipefail + # Usage: https://github.com/DOMjudge/domjudge/blob/main/misc-tools/dj_make_chroot.in#L58-L87 /opt/domjudge/judgehost/bin/dj_make_chroot From 5c995f5a93c9803fd0c961ab317eefb2503bd4de Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Mon, 27 Jul 2020 18:45:32 +0000 Subject: [PATCH 52/61] Switch to headless JDK This is what the manual now suggests, and it's more consistent with dj_make_chroot which uses default-jre-headless. Not updating debian/control and live-image/install.sh for now because they are legacy or currently untested. --- docker-contributor/Dockerfile | 2 +- docker-gitlabci/Dockerfile | 2 +- docker/judgehost/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index d99270cd..084663ca 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -32,7 +32,7 @@ RUN apt-get update \ texlive-latex-recommended texlive-latex-extra \ texlive-fonts-recommended texlive-lang-european latexmk \ sudo debootstrap libcgroup-dev procps \ - default-jre-headless default-jdk \ + default-jre-headless default-jdk-headless \ supervisor apache2-utils lsb-release \ libcurl4-gnutls-dev libjsoncpp-dev libmagic-dev \ enscript lpr ca-certificates less vim \ diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 3e95d80c..ae3f0638 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -5,7 +5,7 @@ ENV APTINSTALL="apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::O RUN apt-get update && apt-get install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ - gcc g++ default-jre-headless default-jdk ghc fp-compiler libcgroup-dev \ + gcc g++ default-jre-headless default-jdk-headless ghc fp-compiler libcgroup-dev \ devscripts shellcheck nginx libboost-regex-dev composer codespell \ php8.3 php8.3-cli php8.3-gd php8.3-curl php8.3-mysql php8.3-gmp php8.3-zip php8.3-xml php8.3-mbstring php8.3-fpm php8.3-intl php8.3-pcov \ # Submit client \ diff --git a/docker/judgehost/Dockerfile b/docker/judgehost/Dockerfile index ea5c22a4..6579fd8a 100644 --- a/docker/judgehost/Dockerfile +++ b/docker/judgehost/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update \ acl lsof zip unzip supervisor sudo procps libcgroup2 \ php-cli php-zip php-gd php-curl php-mysql php-json \ php-gmp php-xml php-mbstring python3 \ - gcc g++ default-jre-headless default-jdk ghc fp-compiler \ + gcc g++ default-jre-headless default-jdk-headless ghc fp-compiler \ && rm -rf /var/lib/apt/lists/* # Add chroot and judgehost data From b6ae2719d4a847dbda9a53910783ec00c59b3544 Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Wed, 10 Jul 2024 13:37:17 +0000 Subject: [PATCH 53/61] Preserve ownership of /opt/domjudge set by `make install-*` Previously, the DOMjudge Docker scripts changed the ownership of /opt/domjudge to "domjudge" recursively, overriding the ownership set by the DOMjudge installation commands (`make install-domserver` and `make install-judgehost`), which mostly set the owner to "root". It is unclear why the Docker scripts did that, since the DOMjudge installation commands should be responsible for installing with the correct ownership. This commit removes the `chown -R` calls from the Docker scripts in order to preserve the ownership set by the DOMjudge installation commands and avoid security issues. Note that the new behaviour is slightly fragile because it relies on Docker's `COPY --from` directive to preserve the ownership when copying files between build stages, and that only works if the numerical user and group IDs are the same. We plan to add a check that the IDs are the same. --- docker/domserver/Dockerfile | 3 +++ docker/domserver/configure.sh | 4 +--- docker/judgehost/Dockerfile | 5 ++++- docker/judgehost/configure.sh | 3 --- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/domserver/Dockerfile b/docker/domserver/Dockerfile index 22051c1e..1ca27649 100644 --- a/docker/domserver/Dockerfile +++ b/docker/domserver/Dockerfile @@ -51,6 +51,9 @@ ENV DEBIAN_FRONTEND=noninteractive \ FPM_MAX_CHILDREN=40 \ DJ_DB_INSTALL_BARE=0 +# Set up user +RUN useradd -m domjudge + # Install required packages for running of domserver RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ diff --git a/docker/domserver/configure.sh b/docker/domserver/configure.sh index 5bd82143..05d8d08d 100644 --- a/docker/domserver/configure.sh +++ b/docker/domserver/configure.sh @@ -1,9 +1,7 @@ #!/bin/sh -eu -# Add user, create PHP FPM socket dir, change permissions for domjudge directory and fix scripts -useradd -m domjudge +# Create PHP FPM socket dir, change permissions for some domjudge directories and fix scripts mkdir -p /run/php -chown -R domjudge: /opt/domjudge chown -R www-data: /opt/domjudge/domserver/tmp # for DOMjudge <= 7.2 (submitdir was removed in commit DOMjudge/domjudge@d66725038) if [ -d /opt/domjudge/domserver/submissions ] diff --git a/docker/judgehost/Dockerfile b/docker/judgehost/Dockerfile index 6579fd8a..bda39f01 100644 --- a/docker/judgehost/Dockerfile +++ b/docker/judgehost/Dockerfile @@ -10,6 +10,9 @@ ENV DEBIAN_FRONTEND=noninteractive \ DOMJUDGE_CREATE_WRITABLE_TEMP_DIR=0 \ RUN_USER_UID_GID=62860 +# Set up user +RUN useradd -m domjudge + # Install required packages for running of judgehost RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ @@ -31,7 +34,7 @@ COPY judgehost/scripts /scripts/ # Make the scripts available to the root user ENV PATH="$PATH:/opt/domjudge/judgehost/bin" -# Change start script permissions, add user and fix permissions +# Run customizations COPY judgehost/configure.sh /configure.sh RUN chmod 700 /configure.sh && /configure.sh && rm -f /configure.sh diff --git a/docker/judgehost/configure.sh b/docker/judgehost/configure.sh index e6766a4e..013487f7 100644 --- a/docker/judgehost/configure.sh +++ b/docker/judgehost/configure.sh @@ -1,8 +1,5 @@ #!/bin/bash -e -useradd -m domjudge -chown -R domjudge: /opt/domjudge - chmod 755 /scripts/start.sh for script in /scripts/bin/* do From 2cb7eb99c729248a78408c713feb8af0d7c989b6 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 4 Aug 2024 12:03:44 +0200 Subject: [PATCH 54/61] Drop support for building containers DOMjudge <8.0 We assume PHP8 already in the scripts, so I suspect this did not work anymore. When someone wants such an older version they can still check the history for the current files and alter the setup from there. --- docker/domserver/build.sh | 10 ---- docker/domserver/configure.sh | 5 -- .../domserver/scripts/start.d/50-domjudge.sh | 60 +------------------ 3 files changed, 2 insertions(+), 73 deletions(-) diff --git a/docker/domserver/build.sh b/docker/domserver/build.sh index cfd41179..14be2d3c 100755 --- a/docker/domserver/build.sh +++ b/docker/domserver/build.sh @@ -8,22 +8,12 @@ sudo -u domjudge ./configure -with-baseurl=http://localhost/ # Passwords should not be included in the built image. We create empty files here to prevent passwords from being generated. sudo -u domjudge touch etc/dbpasswords.secret etc/restapi.secret etc/symfony_app.secret etc/initial_admin_password.secret -if [ ! -f webapp/config/load_db_secrets.php ] -then - # DOMjudge 7.1 - sudo -u domjudge touch webapp/.env.local webapp/.env.local.php -fi sudo -u domjudge make domserver make install-domserver # Remove installed password files rm /opt/domjudge/domserver/etc/*.secret -if [ ! -f webapp/config/load_db_secrets.php ] -then - # DOMjudge 7.1 - rm /opt/domjudge/domserver/webapp/.env.local /opt/domjudge/domserver/webapp/.env.local.php -fi sudo -u domjudge sh -c '. /venv/bin/activate && make docs' # Use Python venv to use the latest Sphinx to build DOMjudge docs. diff --git a/docker/domserver/configure.sh b/docker/domserver/configure.sh index 05d8d08d..a4a5b5ee 100644 --- a/docker/domserver/configure.sh +++ b/docker/domserver/configure.sh @@ -3,11 +3,6 @@ # Create PHP FPM socket dir, change permissions for some domjudge directories and fix scripts mkdir -p /run/php chown -R www-data: /opt/domjudge/domserver/tmp -# for DOMjudge <= 7.2 (submitdir was removed in commit DOMjudge/domjudge@d66725038) -if [ -d /opt/domjudge/domserver/submissions ] -then - chown -R www-data: /opt/domjudge/domserver/submissions -fi chmod 755 /scripts/start.sh for script in /scripts/bin/* diff --git a/docker/domserver/scripts/start.d/50-domjudge.sh b/docker/domserver/scripts/start.d/50-domjudge.sh index dce582ad..64213e72 100755 --- a/docker/domserver/scripts/start.d/50-domjudge.sh +++ b/docker/domserver/scripts/start.d/50-domjudge.sh @@ -42,50 +42,8 @@ else fi # Generate secrets -if [[ -f etc/gen_all_secrets ]] -then - # DOMjudge >= 7.2.1 - (cd etc && ./gen_all_secrets) - # (Note: running 'etc/gen_all_secrets' does not work before commit DOMjudge/domjudge@9bac55144600) -elif [[ -f webapp/config/load_db_secrets.php ]] -then - # DOMjudge 7.2.0 - # This version does not install gen_all_secrets and gensymfonysecret, so we have to inline them here (fixed in commit DOMjudge/domjudge@d523a965f8e0) - if [[ ! -f etc/restapi.secret ]]; then - etc/genrestapicredentials | (umask 077 && cat > etc/restapi.secret) - fi - if [[ ! -f etc/initial_admin_password.secret ]]; then - etc/genadminpassword | (umask 077 && cat > etc/initial_admin_password.secret) - fi - if [[ ! -f etc/symfony_app.secret ]]; then - { - # From etc/gensymfonysecret - head -c20 /dev/urandom | base64 | head -c20 | tr '/+' 'Aa' - echo - } | (umask 077 && cat > etc/symfony_app.secret) - fi -else - # DOMjudge 7.1 - if [[ ! -f etc/restapi.secret ]]; then - etc/genrestapicredentials | (umask 077 && cat > etc/restapi.secret) - fi - if [[ ! -f etc/initial_admin_password.secret ]]; then - etc/genadminpassword | (umask 077 && cat > etc/initial_admin_password.secret) - fi - # This version needs the database settings and app secret to be in webapp/.env.local - # It is generated using etc/gensymfonyenv on DOMjudge 7.1, but that script is not installed so we inline it here - if [[ ! -f webapp/.env.local ]]; then - { - SECRET=$(head -c20 /dev/urandom | base64 | head -c20 | tr '/+' 'Aa') - echo "# Generated on $(hostname), $(date)." - echo - echo "# Uncomment the following line to run the application in development mode" - echo "#APP_ENV=dev" - echo "APP_SECRET=$SECRET" - echo "DATABASE_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DATABASE}" - } | (umask 077 && cat > webapp/.env.local) - fi -fi +(cd etc && ./gen_all_secrets) +# (Note: running 'etc/gen_all_secrets' does not work before commit DOMjudge/domjudge@9bac55144600) # Add the Docker gateway as a trusted proxy if grep -q TRUSTED_PROXIES webapp/.env.local > /dev/null 2>&1 @@ -163,20 +121,6 @@ done echo "real_ip_header X-Forwarded-For;" >> ${NGINX_CONFIG_FILE} echo "real_ip_recursive on;" >> ${NGINX_CONFIG_FILE} -if [[ ! -f webapp/config/load_db_secrets.php ]] -then - # DOMjudge 7.1 dumps the environment into webapp/.env.local.php for improved speed - # We also do that here (with some additional setup to get composer to work) - echo '{"config": {"vendor-dir": "lib/vendor"}, "extra": {"symfony": {"root-dir": "webapp/"}}}' > composer.json - touch webapp/.env - composer symfony:dump-env prod - rm composer.json - if [[ ! -s webapp/.env ]]; then - rm webapp/.env - fi - chmod og= webapp/.env.local.php -fi - # Set up permissions chown -R www-data: webapp/public/images chown www-data: etc/dbpasswords.secret From 3009a1771a4d87c19928871697a7ea2a73616ffd Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Fri, 16 Aug 2024 22:31:32 +0200 Subject: [PATCH 55/61] Install all tools for pa11y as domjudge user The CI broke as the cache only knows of an older version of Chrome. --- docker-gitlabci/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index ae3f0638..55bf92dd 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -22,10 +22,13 @@ RUN apt-get update && apt-get install -y \ npm \ # Code coverage for unit test \ php-pear php-dev \ - # Needed NPM packages \ - && npm install -g pa11y puppeteer \ && rm -rf /var/lib/apt/lists/* +# Install needed NPM packages and download chrome for the domjudge user +RUN su domjudge -c "npm install pa11y puppeteer && \ + npx puppeteer browsers install chrome && \ + npm cache clean" + # Install needed global PHP modules RUN composer -n require justinrainbow/json-schema @@ -48,9 +51,6 @@ RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-0 RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-1 RUN groupadd domjudge-run -# Download chrome for the domjudge user -RUN su domjudge -c "npx puppeteer browsers install chrome" - # Update dictionary to latest version ENV GITHUBDICTIONARY="https://raw.githubusercontent.com/codespell-project/codespell/master/codespell_lib/data/dictionary.txt" ENV CODESPELLDICTLOCATION="/usr/lib/python3/dist-packages/codespell_lib/data/dictionary.txt" From 9c84b131f4d541bba6ec65591f9be7502d72ce84 Mon Sep 17 00:00:00 2001 From: MCJ Vasseur <14887731+vmcj@users.noreply.github.com> Date: Fri, 2 Aug 2024 13:10:03 +0200 Subject: [PATCH 56/61] Trigger on PRs --- .github/workflows/build-contributor-container-PR.yml | 4 ++-- .github/workflows/build-domjudge-container-PR.yml | 4 ++-- .github/workflows/build-gitlab-container-PR.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-contributor-container-PR.yml b/.github/workflows/build-contributor-container-PR.yml index ecd2e2c5..f1c6cab6 100644 --- a/.github/workflows/build-contributor-container-PR.yml +++ b/.github/workflows/build-contributor-container-PR.yml @@ -5,13 +5,13 @@ name: 'Build Contributor container (PR)' on: push: paths: - - docker-contributor + - docker-contributor/** - .github/workflows/build-contributor-container-PR.yml pull_request: branches: - main paths: - - docker-contributor + - docker-contributor/** - .github/workflows/build-contributor-container-PR.yml jobs: diff --git a/.github/workflows/build-domjudge-container-PR.yml b/.github/workflows/build-domjudge-container-PR.yml index 0f5403d7..ca9e5c92 100644 --- a/.github/workflows/build-domjudge-container-PR.yml +++ b/.github/workflows/build-domjudge-container-PR.yml @@ -4,13 +4,13 @@ on: push: paths: - .github/workflows/build-domjudge-container-PR.yml - - docker + - docker/** pull_request: branches: - main paths: - .github/workflows/build-domjudge-container-PR.yml - - docker + - docker/** env: DOMJUDGE_VERSION: M.m.p diff --git a/.github/workflows/build-gitlab-container-PR.yml b/.github/workflows/build-gitlab-container-PR.yml index 776830c1..662580e1 100644 --- a/.github/workflows/build-gitlab-container-PR.yml +++ b/.github/workflows/build-gitlab-container-PR.yml @@ -4,14 +4,14 @@ on: push: paths: - docker-contributor/php-config - - docker-gitlabci + - docker-gitlabci/** - .github/workflows/build-gitlab-container-PR.yml pull_request: branches: - main paths: - docker-contributor/php-config - - docker-gitlabci + - docker-gitlabci/** - .github/workflows/build-gitlab-container-PR.yml jobs: From 55881c81364cc28034a0b9145048665e1e8fdad5 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:22:58 +0200 Subject: [PATCH 57/61] Create the user before installing the tools as that user Also actually cleanup the cache, there is another better method for that but I'll leave that for another PR. --- docker-gitlabci/Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 55bf92dd..84ddcfe8 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -2,6 +2,11 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive ENV PHPSUPPORTED="7.4 8.0 8.1 8.2" ENV APTINSTALL="apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" +RUN useradd -m domjudge +RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-0 +RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-1 +RUN groupadd domjudge-run + RUN apt-get update && apt-get install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ @@ -25,9 +30,10 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* # Install needed NPM packages and download chrome for the domjudge user -RUN su domjudge -c "npm install pa11y puppeteer && \ +RUN su domjudge -c "cd /home/domjudge && \ + npm install pa11y puppeteer && \ npx puppeteer browsers install chrome && \ - npm cache clean" + npm cache clean --force" # Install needed global PHP modules RUN composer -n require justinrainbow/json-schema @@ -46,11 +52,6 @@ RUN add-apt-repository ppa:ondrej/php -y && apt-get update && \ # Put the gitlab user in sudo RUN echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -RUN useradd -m domjudge -RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-0 -RUN useradd -d /nonexistent -g nogroup -s /bin/false domjudge-run-1 -RUN groupadd domjudge-run - # Update dictionary to latest version ENV GITHUBDICTIONARY="https://raw.githubusercontent.com/codespell-project/codespell/master/codespell_lib/data/dictionary.txt" ENV CODESPELLDICTLOCATION="/usr/lib/python3/dist-packages/codespell_lib/data/dictionary.txt" From 914a5014605ded2fbcb9802d9c452f68528aa601 Mon Sep 17 00:00:00 2001 From: Michael Vasseur Date: Sun, 20 Oct 2024 12:23:24 +0200 Subject: [PATCH 58/61] Install tools for dumping the database --- docker-gitlabci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index 84ddcfe8..cab673ca 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update && apt-get install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ gcc g++ default-jre-headless default-jdk-headless ghc fp-compiler libcgroup-dev \ - devscripts shellcheck nginx libboost-regex-dev composer codespell \ + devscripts shellcheck nginx libboost-regex-dev composer codespell pv \ php8.3 php8.3-cli php8.3-gd php8.3-curl php8.3-mysql php8.3-gmp php8.3-zip php8.3-xml php8.3-mbstring php8.3-fpm php8.3-intl php8.3-pcov \ # Submit client \ python3-requests python3-magic \ From 4983ddcddc347e69d814ee06b8a72fbd29b522db Mon Sep 17 00:00:00 2001 From: MCJ Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sun, 20 Oct 2024 15:26:26 +0200 Subject: [PATCH 59/61] Alternative tools for import of example problems --- docker-gitlabci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-gitlabci/Dockerfile b/docker-gitlabci/Dockerfile index cab673ca..0ce0c493 100644 --- a/docker-gitlabci/Dockerfile +++ b/docker-gitlabci/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update && apt-get install -y \ acl make zip unzip apache2-utils bsdmainutils libcurl4-gnutls-dev \ libjsoncpp-dev libmagic-dev autoconf automake bats sudo debootstrap procps \ gcc g++ default-jre-headless default-jdk-headless ghc fp-compiler libcgroup-dev \ - devscripts shellcheck nginx libboost-regex-dev composer codespell pv \ + devscripts shellcheck nginx libboost-regex-dev composer codespell pv httpie \ php8.3 php8.3-cli php8.3-gd php8.3-curl php8.3-mysql php8.3-gmp php8.3-zip php8.3-xml php8.3-mbstring php8.3-fpm php8.3-intl php8.3-pcov \ # Submit client \ python3-requests python3-magic \ From 628381697202122168e7e227e36a7562a492471f Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Sun, 13 Oct 2024 16:02:58 +0200 Subject: [PATCH 60/61] Add Apache2 configuration to docker-contributor Adding configuration for Apache2 enables easier testing of webserver-specific features and issues. By default, the contributor image still uses NGINX as webserver. Add an option to use Apache2 by default, or switch back and forth between NGINX/Apache2 with the `switch-webserver` command. --- docker-contributor/Dockerfile | 1 + docker-contributor/README.md | 8 ++++-- .../scripts/bin/apache2-access-log | 2 ++ .../scripts/bin/apache2-error-log | 2 ++ .../scripts/bin/switch-webserver | 14 ++++++++++ docker-contributor/scripts/start.sh | 27 +++++++++++++++++++ docker-contributor/supervisor/apache2.conf | 5 ++++ docker-contributor/supervisor/nginx.conf | 2 +- 8 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 docker-contributor/scripts/bin/apache2-access-log create mode 100644 docker-contributor/scripts/bin/apache2-error-log create mode 100644 docker-contributor/scripts/bin/switch-webserver create mode 100644 docker-contributor/supervisor/apache2.conf diff --git a/docker-contributor/Dockerfile b/docker-contributor/Dockerfile index 084663ca..3be8a7ce 100644 --- a/docker-contributor/Dockerfile +++ b/docker-contributor/Dockerfile @@ -16,6 +16,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ DJ_DB_INSTALL_BARE=0 \ PHPSUPPORTED="8.1 8.2 8.3" \ DEFAULTPHPVERSION="8.3" \ + DEFAULTWEBSERVER="nginx" \ APTINSTALL="apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" # Install required packages and clean up afterwards to make this image layer smaller diff --git a/docker-contributor/README.md b/docker-contributor/README.md index cd265fea..41324f83 100644 --- a/docker-contributor/README.md +++ b/docker-contributor/README.md @@ -9,7 +9,7 @@ The container includes the following: * Set up or update the database. * Set up the webserver. * Create a chroot. -* PHP-FPM and nginx for running the web interface. +* PHP-FPM and apache2 or nginx for running the web interface. * Two running judgedaemons using a chroot. * Scripts for reading the log files of the webserver and the judgedaemons. * A script to create a dummy DOMjudge user and submit all test submissions. @@ -67,6 +67,7 @@ The following environment variables are supported by the container: * `MYSQL_DATABASE` (defaults to `domjudge`): set the database to use. * `FPM_MAX_CHILDREN` (defaults to `40`): the maximum number of PHP FPM children to spawn. * `DJ_SKIP_MAKE` (defaults to `0`): set to `1` to skip the maintainer setup and install commands. This will speed up the startup process of the container and is useful if this is already done before. +* `DEFAULTWEBSERVER` (defaults to `nginx`): set to `apache2` to use the Apache2 httpd server as default webserver. #### Passwords through files @@ -97,6 +98,8 @@ If you have named your container something other than `domjudge`, be sure to cha The following commands are available: +* `apache2-access-log`: tail the access log of apache2. +* `apache2-error-log`: tail the error log of apache2. * `nginx-access-log`: tail the access log of nginx. * `nginx-error-log`: tail the error log of nginx. * `judgedaemon-log 0` and `judgedaemon-log 1`: tail the log of the first / second judgeaemon. @@ -105,6 +108,7 @@ The following commands are available: * `xdebug-enable`: enable Xdebug debugging. See note below * `xdebug-disable`: disable Xdebug debugging. See note below * `switch-php `: switch to using the given PHP version. +* `switch-webserver `: switch to using the given webserver. Of course, you can always run `docker exec -it domjudge bash` to get a bash shell inside the container. @@ -114,7 +118,7 @@ To restart any of the services, run the following: docker exec -it domjudge supervisorctl restart [service] ``` -where `[service]` is one of `nginx`, `php`, `judgedaemon0` or `judgedaemon1`. +where `[service]` is one of `apache2`, `nginx`, `php`, `judgedaemon0` or `judgedaemon1`. ### Xdebug diff --git a/docker-contributor/scripts/bin/apache2-access-log b/docker-contributor/scripts/bin/apache2-access-log new file mode 100644 index 00000000..418d6c1b --- /dev/null +++ b/docker-contributor/scripts/bin/apache2-access-log @@ -0,0 +1,2 @@ +#!/bin/bash +supervisorctl tail -f apache2 diff --git a/docker-contributor/scripts/bin/apache2-error-log b/docker-contributor/scripts/bin/apache2-error-log new file mode 100644 index 00000000..be78e366 --- /dev/null +++ b/docker-contributor/scripts/bin/apache2-error-log @@ -0,0 +1,2 @@ +#!/bin/bash +supervisorctl tail -f apache2 stderr diff --git a/docker-contributor/scripts/bin/switch-webserver b/docker-contributor/scripts/bin/switch-webserver new file mode 100644 index 00000000..7db0184c --- /dev/null +++ b/docker-contributor/scripts/bin/switch-webserver @@ -0,0 +1,14 @@ +#!/bin/bash +WEBSERVER=$1 +if [ "${WEBSERVER}" = "nginx" ] +then + sudo supervisorctl stop apache2 + sudo supervisorctl start nginx +elif [ "${WEBSERVER}" = "apache2" ] +then + sudo supervisorctl stop nginx + sudo supervisorctl start apache2 +else + echo "Usage: $0 [apache2|nginx]" + exit 1 +fi diff --git a/docker-contributor/scripts/start.sh b/docker-contributor/scripts/start.sh index 06e27f5b..e1866cf2 100644 --- a/docker-contributor/scripts/start.sh +++ b/docker-contributor/scripts/start.sh @@ -149,6 +149,23 @@ sudo sed -i '/error_log/d' $NGINX_CONFIG_FILE # Use debug front controller sudo sed -i 's/app\.php/app_dev.php/g' $NGINX_CONFIG_FILE sudo sed -i 's/app\\\.php/app\\_dev.php/g' $NGINX_CONFIG_FILE + +# Configure Apache2 +APACHE2_CONFIG_FILE=/etc/apache2/conf-available/domjudge.conf +sudo cp etc/apache.conf $APACHE2_CONFIG_FILE +sudo a2enmod proxy_fcgi setenvif rewrite +sudo cp "/etc/apache2/conf-available/php$DEFAULTPHPVERSION-fpm.conf" /etc/apache2/conf-available/php-domjudge-fpm.conf +sudo sed -i 's/proxy:unix:.*|/proxy:unix:\/var\/run\/php-fpm-domjudge.sock|/' /etc/apache2/conf-available/php-domjudge-fpm.conf +sudo a2enconf php-domjudge-fpm domjudge +sudo rm /etc/apache2/sites-enabled/000-default.conf +# Run DOMjudge in root +sudo sed -i '/^#/,/^#<\/VirtualHost>/ s/#//' $APACHE2_CONFIG_FILE +sudo sed -i 's/^Alias \/domjudge/#Alias \/domjudge/' $APACHE2_CONFIG_FILE +# Run as user and group 'domjudge' +sudo sed -i 's//User domjudge\nGroup domjudge\n/' $APACHE2_CONFIG_FILE +# Redirect logs to stdout/stderr +sudo sed -i 's//TransferLog \/dev\/stdout\nErrorLog \/dev\/stderr\n/' $APACHE2_CONFIG_FILE + # Set up permissions (make sure the script does not stop if this fails, as this will happen on macOS / Windows) sudo chown domjudge: "${PROJECT_DIR}/webapp/var" echo "[ok] Webserver config installed"; echo @@ -176,4 +193,14 @@ echo "[ok] Sudoers configuration added"; echo sudo sed -i "s|PROJECT_DIR|${PROJECT_DIR}|" /etc/supervisor/conf.d/judgedaemon.conf sudo sed -i "s|PROJECT_DIR|${PROJECT_DIR}|" /etc/supervisor/conf.d/judgedaemonextra.conf +echo "[..] Configuring default webserver" +if [ "${DEFAULTWEBSERVER}" = "apache2" ] || [ "${DEFAULTWEBSERVER}" = "nginx" ] +then + sudo sed -i "s|autostart=false|autostart=true|" "/etc/supervisor/conf.d/$DEFAULTWEBSERVER.conf" +else + echo "Unsupported webserver '$DEFAULTWEBSERVER'" + exit 1 +fi +echo "[ok] Configured default webserver"; echo + exec sudo supervisord -n -c /etc/supervisor/supervisord.conf diff --git a/docker-contributor/supervisor/apache2.conf b/docker-contributor/supervisor/apache2.conf new file mode 100644 index 00000000..cd8eba5d --- /dev/null +++ b/docker-contributor/supervisor/apache2.conf @@ -0,0 +1,5 @@ +[program:apache2] +command=pidproxy /var/run/apache2/apache2.pid /bin/bash -c "source /etc/apache2/envvars && apache2ctl -D FOREGROUND" +numprocs=1 +autostart=false +autorestart=true diff --git a/docker-contributor/supervisor/nginx.conf b/docker-contributor/supervisor/nginx.conf index 4ac205a1..efebb34c 100644 --- a/docker-contributor/supervisor/nginx.conf +++ b/docker-contributor/supervisor/nginx.conf @@ -1,5 +1,5 @@ [program:nginx] command=nginx -g "daemon off;" numprocs=1 -autostart=true +autostart=false autorestart=true From b29136832ceefd71092fa55eed58a3157476eefd Mon Sep 17 00:00:00 2001 From: Kevin Jilissen Date: Tue, 22 Oct 2024 13:51:49 +0200 Subject: [PATCH 61/61] Allow deletion of default config to fail If the container is restarted, the configuration file does not exist. Do not fail the `rm` command if the file could not be removed to ensure idempotency of the Apache2 configuration part. Fixes an issue introduced in 628381697202122168e7e227e36a7562a492471f. --- docker-contributor/scripts/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-contributor/scripts/start.sh b/docker-contributor/scripts/start.sh index e1866cf2..1b085741 100644 --- a/docker-contributor/scripts/start.sh +++ b/docker-contributor/scripts/start.sh @@ -157,7 +157,7 @@ sudo a2enmod proxy_fcgi setenvif rewrite sudo cp "/etc/apache2/conf-available/php$DEFAULTPHPVERSION-fpm.conf" /etc/apache2/conf-available/php-domjudge-fpm.conf sudo sed -i 's/proxy:unix:.*|/proxy:unix:\/var\/run\/php-fpm-domjudge.sock|/' /etc/apache2/conf-available/php-domjudge-fpm.conf sudo a2enconf php-domjudge-fpm domjudge -sudo rm /etc/apache2/sites-enabled/000-default.conf +sudo rm -f /etc/apache2/sites-enabled/000-default.conf # Run DOMjudge in root sudo sed -i '/^#/,/^#<\/VirtualHost>/ s/#//' $APACHE2_CONFIG_FILE sudo sed -i 's/^Alias \/domjudge/#Alias \/domjudge/' $APACHE2_CONFIG_FILE