From 5324cb17a3e363575a214a836c0dff38848288d0 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Fri, 29 Dec 2023 10:54:31 +0100 Subject: [PATCH] Adjust configuration to Docker Compose V2 (#186) This patch replaces uses of `docker-compose` with `docker compose`. If the new command doesn't exist we still fall back to `docker-compose`. --- README.md | 2 +- deployment/Makefile | 21 ++++++++++++--------- deployment/README.md | 2 +- deployment/docker-compose.yml | 1 - loadbalancer/Makefile | 17 ++++++++++------- test/perf/Makefile | 7 +++++-- 6 files changed, 29 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 5ab94a3..b5657de 100644 --- a/README.md +++ b/README.md @@ -19,4 +19,4 @@ $ JULIA_PKG_SERVER=http://localhost:8000 julia ## Deployment -See the [deployment](deployment) directory for an example `docker-compose` setup to easily deploy your own mirror Pkg server, with an optional SSL interface provided through `nginx` and `letsencrypt`. +See the [deployment](deployment) directory for an example `docker compose` setup to easily deploy your own mirror Pkg server, with an optional SSL interface provided through `nginx` and `letsencrypt`. diff --git a/deployment/Makefile b/deployment/Makefile index 4353b46..5a5ae4d 100644 --- a/deployment/Makefile +++ b/deployment/Makefile @@ -1,6 +1,9 @@ # Default target is to bring everything up all: up +# Use docker compose if it exist, fall back to docker-compose +COMPOSE=$(shell if docker compose version >/dev/null 2>&1; then echo "docker compose"; else echo "docker-compose"; fi) + # Extract configuration from `.env` file PKG_SERVER_REGION := $(shell bash -c "source .env 2>/dev/null && echo \$$PKG_SERVER_REGION") ifeq ($(PKG_SERVER_REGION),) @@ -60,13 +63,13 @@ build/pkgserver.nginx.conf: conf/pkgserver.nginx.conf ${LISTEN_SRC} | build envsubst '$${LISTEN_BLOCK_SRC} $${LISTEN_BLOCK} $${MIRROR_BLOCK} $${REGION} $${FQDN}' < $< > $@ up: $(DIRS) build/pkgserver.nginx.conf - docker-compose up --build --remove-orphans -d + ${COMPOSE} up --build --remove-orphans -d down: - docker-compose down --remove-orphans + ${COMPOSE} down --remove-orphans pull: - docker-compose pull + ${COMPOSE} pull # Rule to install logrotate configuration so that host-wide logrotate can # perform the logrotations, as well as run `make nginx-send-usr1`. @@ -84,9 +87,9 @@ include logrotate.make sudo chmod +x $@ up: /etc/cron.hourly/pkgserver -# Rule to send nginx USR1 signal via docker-compose (used by `logrotate.conf`) +# Rule to send nginx USR1 signal via docker compose (used by `logrotate.conf`) nginx-send-usr1: - docker-compose exec -T frontend /bin/bash -c 'kill -USR1 `pgrep -f "nginx: [m]aster"`' + ${COMPOSE} exec -T frontend /bin/bash -c 'kill -USR1 `pgrep -f "nginx: [m]aster"`' /etc/cron.daily/pkgserver: conf/cron.restart.conf SOURCEDIR=$(shell pwd) PYTHONPATH=$(PYTHONPATH) UID=$(UID) GID=$(GID) envsubst < $< | sudo tee $@ >/dev/null @@ -109,17 +112,17 @@ dev-up: stop-watchtower: - docker-compose stop watchtower + ${COMPOSE} stop watchtower trigger-watchtower: - docker-compose run --rm watchtower --cleanup --scope pkgserver.jl --run-once + ${COMPOSE} run --rm watchtower --cleanup --scope pkgserver.jl --run-once logs: - docker-compose logs -f --tail=100 + ${COMPOSE} logs -f --tail=100 .PHONY: logs destroy: - docker-compose down -v --remove-orphans + ${COMPOSE} down -v --remove-orphans rm -rf logs storage build sudo rm -f /etc/cron.hourly/pkgserver /etc/cron.daily/pkgserver diff --git a/deployment/README.md b/deployment/README.md index 8926506..44f7559 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -2,6 +2,6 @@ Deployment of a new PkgServer is as easy as filling out a `.env` file and running `make`. The `.env` file controls the customization of the PkgServer and associated containers. The values that should be set are listed in `.env.example`, along with whether the value is required or not. There are two separate `docker-compose.yml` configurations listed here; a non-SSL version, and an SSL version. The non-SSL version just runs the pkg server directly on port `8000`, while the SSL version runs an `nginx` SSL terminator (hooked up with `letsencrypt`), then `proxy_pass`'es the decrypted traffic to a PkgServer instance running within a separate container. -To avoid needing to memorize arcane `docker-compose` commands, a handy `Makefile` is provided to ease usage of `docker-compose`. Valid make verbs are `up`, `build`, `logs`, `down`, `destroy`, etc.... By default, the non-SSL version will be deployed, to build/deploy the SSL version, invoke `make` with the `USE_SSL=1` flag. +To avoid needing to memorize arcane `docker compose` commands, a handy `Makefile` is provided to ease usage of `docker compose`. Valid make verbs are `up`, `build`, `logs`, `down`, `destroy`, etc.... By default, the non-SSL version will be deployed, to build/deploy the SSL version, invoke `make` with the `USE_SSL=1` flag. The deployment also includes a [`watchtower`](https://github.com/containrrr/watchtower) listener that polls Docker Hub for updated versions of the [`juliapackaging/pkgserver.jl`](https://hub.docker.com/r/juliapackaging/pkgserver.jl) image every 5 minutes. When a new version is discovered, it is automatically pulled down. This forms the basis of our automatic deployment pipeline; as a PR is merged within this repository, it will trigger a new build of the `pkgserver.jl` docker image, which then gets downloaded by any and all deployments within a matter of minutes. In the event of catastrophic server failure (e.g. a bug slips past and the PkgServer cannot start) pushing a new version should robustly upgrade, as the `watchtower` image is separated from the `pkgserver` process. diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index d0e33e6..e4541e2 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -14,7 +14,6 @@ services: - ./logs/pkgserver:/app/logs environment: JULIA_PKG_SERVER: "0.0.0.0:8000" - JULIA_PKG_SERVER_STORAGE_SERVERS: "https://us-east.storage.juliahub.com,https://kr.storage.juliahub.com" JULIA_PKG_SERVER_STORAGE_ROOT: "/app/storage" JULIA_PKG_SERVER_LOGS_DIR: "/app/logs" JULIA_PKG_SERVER_FQDN: "${PKG_SERVER_FQDN}" diff --git a/loadbalancer/Makefile b/loadbalancer/Makefile index 633b215..28bd3ee 100644 --- a/loadbalancer/Makefile +++ b/loadbalancer/Makefile @@ -5,6 +5,9 @@ all: up COMMA:=, SPACE:=$(eval) $(eval) +# Use docker compose if it exist, fall back to docker-compose +COMPOSE=$(shell if docker compose version >/dev/null 2>&1; then echo "docker compose"; else echo "docker-compose"; fi) + # Extract configuration from `.env` file SERVER_REGION := $(shell bash -c "source .env 2>/dev/null && echo \$$SERVER_REGION") PKGSERVERS := $(shell bash -c "source .env 2>/dev/null && echo \$$PKGSERVERS") @@ -33,10 +36,10 @@ build/children.json: .env | build up: build/children.json up: $(DIRS) build/loadbalancer.nginx.conf - docker-compose up --build --remove-orphans -d + ${COMPOSE} up --build --remove-orphans -d down: - docker-compose down --remove-orphans + ${COMPOSE} down --remove-orphans # Rule to install logrotate configuration so that host-wide logrotate can # perform the logrotations, as well as run `make nginx-send-usr1`. @@ -54,19 +57,19 @@ up: /etc/cron.hourly/loadbalancer include ../deployment/logrotate.make -# Rule to send nginx USR1 signal via docker-compose (used by `logrotate.conf`) +# Rule to send nginx USR1 signal via docker compose (used by `logrotate.conf`) nginx-send-usr1: - docker-compose exec -T loadbalancer /bin/bash -c 'kill -USR1 `pgrep -f "nginx: [m]aster"`' + ${COMPOSE} exec -T loadbalancer /bin/bash -c 'kill -USR1 `pgrep -f "nginx: [m]aster"`' stop-watchtower: - docker-compose stop watchtower + ${COMPOSE} stop watchtower logs: - docker-compose logs -f --tail=100 + ${COMPOSE} logs -f --tail=100 .PHONY: logs destroy: - docker-compose down -v --remove-orphans + ${COMPOSE} down -v --remove-orphans rm -rf logs build restart: diff --git a/test/perf/Makefile b/test/perf/Makefile index ffea062..f4b1879 100644 --- a/test/perf/Makefile +++ b/test/perf/Makefile @@ -1,12 +1,15 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +# Use docker compose if it exist, fall back to docker-compose +COMPOSE=$(shell if docker compose version >/dev/null 2>&1; then echo "docker compose"; else echo "docker-compose"; fi) + K6 := docker run --rm -v $(SRCDIR):/config:ro -v $(SRCDIR)/results:/results --net=host -ti loadimpact/k6 up: - docker-compose up -d --remove-orphans + ${COMPOSE} up -d --remove-orphans down: - docker-compose down -v --remove-orphans + ${COMPOSE} down -v --remove-orphans results: mkdir -p $@