Merge pull request #7 from mrc-ide/mrc-5146-container-gh-actions

Mrc 5146 container gh actions

.github/workflows/build-and-push.yaml

@@ -0,0 +1,65 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+name: Build-and-push
+
+env:
+  TAG_DH: mrcide/orderly.runner
+  TAG_GHCR: mrc-ide/orderly.runner
+  GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    # Shorter timeout to prevent mac builders hanging for 6 hours!
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - id: ci-env
+        name: Setup Environment
+        shell: bash
+        run: |
+          if [ "${{github.event_name}}" = "pull_request" ];
+          then
+            long_sha=${{ github.event.pull_request.head.sha }}
+            echo "CI_BRANCH=${{ github.head_ref }}" >> $GITHUB_OUTPUT
+          else
+            long_sha=${GITHUB_SHA}
+            echo "CI_BRANCH=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+          fi
+          echo "CI_SHA=${long_sha:0:7}" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GHCR (GitHub Packages)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ env.GITHUB_PAT }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          file: ./docker/Dockerfile
+          tags: |
+            ${{env.TAG_DH}}:${{steps.ci-env.outputs.CI_SHA}}
+            ${{env.TAG_DH}}:${{steps.ci-env.outputs.CI_BRANCH}}
+            ghcr.io/${{env.TAG_GHCR}}:${{steps.ci-env.outputs.CI_SHA}}
+            ghcr.io/${{env.TAG_GHCR}}:${{steps.ci-env.outputs.CI_BRANCH}}

DESCRIPTION

@@ -12,6 +12,8 @@ RoxygenNote: 7.3.1
 Roxygen: list(markdown = TRUE, roclets = c("rd", "namespace", "porcelain::porcelain_roclet"))
 URL: https://github.com/mrc-ide/orderly.runner
 BugReports: https://github.com/mrc-ide/orderly.runner/issues
+Depends:
+    R (>= 4.1.0)
 Imports:
     cli,
     docopt,

R/main.R

@@ -1,20 +1,47 @@
 parse_main <- function(args = commandArgs(TRUE)) {
   usage <- "Usage:
-orderly.runner [options] <path>
+orderly.runner.server [options] <path>
 
 Options:
   --log-level=LEVEL  Log-level (off, info, all) [default: info]
   --validate         Enable json schema validation
+  --host=HOST        Host to run api on [default: 0.0.0.0]
   --port=PORT        Port to run api on [default: 8001]"
   dat <- docopt::docopt(usage, args)
   list(log_level = dat$log_level,
        validate = dat$validate,
        port = as.integer(dat$port),
-       path = dat$path)
+       path = dat$path,
+       host = dat$host)
 }
 
-
 main <- function(args = commandArgs(TRUE)) {
   dat <- parse_main(args)
-  api(dat$path, dat$validate, dat$log_level)$run("0.0.0.0", port = dat$port)
+  api_obj <- api(dat$path, dat$validate, dat$log_level)
+  api_obj$run(host = dat$host, port = dat$port)
+}
+
+
+parse_main_worker <- function(args = commandArgs(TRUE)) {
+  usage <- "Usage:
+orderly.runner.worker <path>"
+  dat <- docopt::docopt(usage, args)
+  list(path = dat$path)
+}
+
+main_worker <- function(args = commandArgs(TRUE)) {
+  dat <- parse_main_worker(args)
+
+  # assumes ORDERLY_RUNNER_QUEUE_ID is set
+  queue <- Queue$new(dat$path)
+
+  worker <- rrq::rrq_worker$new(
+    queue$controller$queue_id,
+    con = queue$controller$con
+  )
+  worker_path <- file.path(dat$path, ".packit", "workers", worker$id)
+  fs::dir_create(worker_path)
+  gert::git_clone(dat$path, path = worker_path)
+
+  worker$loop()
 }

R/queue.R

@@ -25,9 +25,13 @@ Queue <- R6::R6Class("Queue", #nolint
                              "root is not version controlled."))
       }
 
+      # Connect to Redis
+      con <- redux::hiredis(host = redis_host())
+
       # Create queue
       self$controller <- rrq::rrq_controller(
-        queue_id %||% orderly_queue_id()
+        queue_id %||% orderly_queue_id(),
+        con = con
       )
       worker_config <- rrq::rrq_worker_config(heartbeat_period = 10)
       rrq::rrq_worker_config_save("localhost", worker_config,
@@ -77,3 +81,8 @@ orderly_queue_id <- function() {
   id <- Sys.getenv("ORDERLY_RUNNER_QUEUE_ID", "")
   if (nzchar(id)) id else sprintf("orderly.runner:%s", ids::random_id())
 }
+
+redis_host <- function() {
+  name <- Sys.getenv("REDIS_CONTAINER_NAME", "")
+  if (nzchar(name)) name else NULL
+}

README.md

@@ -15,6 +15,21 @@ To install `orderly.runner`:
 remotes::install_github("mrc-ide/orderly.runner", upgrade = FALSE)
 ```
 
+## Testing and development
+
+To run the full docker setup:
+
+1. Optionally modify `docker/test/examples` orderly reports. These will be used as reports in the container
+1. Run `docker/test/run-test` - this will produce a `test-repo` directory to show you what was copied into the docker containers (you can create just this directory without running the docker containers by running `docker/test/setup-test-repo` if you want)
+1. The server will be available at `localhost:8001`
+1. To view the orderly root directory in the docker container (you may want to do this after workers have run orderly reports for example), run `docker/test/copy-orderly-root` and this will copy the contents to `docker/test/orderly-root-volume`
+1. Finally to clear docker and remove `test-repo` and `orderly-root-volume` directories run `docker/test/clear-test`
+
+
+## Notes for deploying
+
+When running the server or worker containers, you should have `REDIS_CONTAINER_NAME` env var set to connect to the redis container from the server and worker containers. You should also set the `ORDERLY_RUNNER_QUEUE_ID` to the same thing between server and worker containers so they connect to the same queue.
+
 ## License
 
 MIT © Imperial College of Science, Technology and Medicine

docker/Dockerfile

@@ -1,7 +1,9 @@
-FROM rocker/r-ver:4.0.4
+FROM rocker/r-ver:4.1
 
 RUN apt-get update &&  apt-get install -y --no-install-recommends \
+        git \
         libcurl4-openssl-dev \
+        libhiredis-dev \
         libssl-dev \
         zlib1g-dev \
     && apt-get clean \
@@ -17,3 +19,12 @@ RUN install2.r --error remotes &&  \
 
 COPY . /src
 RUN R CMD INSTALL --install-tests /src && rm -rf /src
+
+COPY docker/bin /usr/local/bin/
+
+RUN git config --global --add safe.directory "*"
+RUN echo ".packit" > /.gitignore
+RUN git config --global core.excludesFile "/.gitignore"
+
+# ENTRYPOINT for server is /usr/local/bin/orderly.runner.server
+# ENTRYPOINT for worker is /usr/local/bin/orderly.runner.worker

docker/bin/orderly.runner.server

@@ -0,0 +1,2 @@
+#!/usr/bin/env Rscript
+orderly.runner:::main()

docker/bin/orderly.runner.worker

@@ -0,0 +1,2 @@
+#!/usr/bin/env Rscript
+orderly.runner:::main_worker()

docker/test/.gitignore

@@ -0,0 +1,2 @@
+test-repo
+orderly-root-volume

docker/test/clear-test

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -x
+HERE=$(dirname $0)
+
+. $HERE/common
+
+docker rm -f \
+    $REDIS \
+    $SERVER \
+    $WORKER \
+    $DEBUG
+
+docker network rm $NETWORK
+
+docker volume rm $ORDERLY_VOLUME
+
+rm -rf $HERE/test-repo
+rm -rf $HERE/orderly-root-volume

docker/test/common

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exu
+
+export CONTAINER_NAMESPACE=orderly.runner
+
+export NETWORK=$CONTAINER_NAMESPACE-network
+export REDIS=$CONTAINER_NAMESPACE-redis
+export DEBUG=$CONTAINER_NAMESPACE-debug
+export SERVER=$CONTAINER_NAMESPACE-server
+export WORKER=$CONTAINER_NAMESPACE-worker
+
+export ORDERLY_RUNNER_QUEUE_ID=orderly.runner.queue
+
+export ORDERLY_VOLUME=$CONTAINER_NAMESPACE-orderly-root
+
+export ORDERLY_RUNNER_IMAGE=ghcr.io/mrc-ide/orderly.runner:main
+
+export CONTAINER_ORDERLY_ROOT_PATH=/orderly-root

docker/test/copy-orderly-root

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -x
+HERE=$(dirname $0)
+
+. $HERE/common
+
+rm -rf $HERE/orderly-root-volume
+mkdir -p $HERE/orderly-root-volume
+
+docker cp $DEBUG:$CONTAINER_ORDERLY_ROOT_PATH/. $HERE/orderly-root-volume

docker/test/examples/data/data.R

@@ -0,0 +1,2 @@
+data <- list(a = c(1, 2))
+write.csv(data, "data.csv")

docker/test/examples/parameters/parameters.R

@@ -0,0 +1,3 @@
+orderly2::orderly_parameters(a = NULL)
+data <- list(a = a)
+write.csv(data, "parameters.csv")

docker/test/run-test

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -exu
+HERE=$(dirname $0)
+ABSOLUTE_PATH=$(realpath $HERE)
+
+. $HERE/common
+
+$HERE/setup-test-repo
+
+docker network create $NETWORK
+
+docker run --rm -d \
+       --net=$NETWORK \
+       --name=$REDIS \
+       -p 127.0.0.1:6379:6379 \
+       redis
+
+docker volume create $ORDERLY_VOLUME
+
+docker run --rm -d --pull=missing \
+       --name=$DEBUG \
+       -v $ORDERLY_VOLUME:$CONTAINER_ORDERLY_ROOT_PATH \
+       ubuntu \
+       sleep infinity
+
+docker cp $ABSOLUTE_PATH/test-repo/. $DEBUG:$CONTAINER_ORDERLY_ROOT_PATH
+
+docker run --rm -d --pull=always \
+       --net=$NETWORK \
+       --name=$SERVER \
+       --entrypoint="/usr/local/bin/orderly.runner.server" \
+       --env=ORDERLY_RUNNER_QUEUE_ID=$ORDERLY_RUNNER_QUEUE_ID \
+       --env=REDIS_CONTAINER_NAME=$REDIS \
+       -p 127.0.0.1:8001:8001 \
+       -v $ORDERLY_VOLUME:$CONTAINER_ORDERLY_ROOT_PATH \
+       $ORDERLY_RUNNER_IMAGE \
+       $CONTAINER_ORDERLY_ROOT_PATH
+
+docker run --rm -d --pull=always \
+       --net=$NETWORK \
+       --name=$WORKER \
+       --entrypoint="/usr/local/bin/orderly.runner.worker" \
+       --env=ORDERLY_RUNNER_QUEUE_ID=$ORDERLY_RUNNER_QUEUE_ID \
+       --env=REDIS_CONTAINER_NAME=$REDIS \
+       -v $ORDERLY_VOLUME:$CONTAINER_ORDERLY_ROOT_PATH \
+       $ORDERLY_RUNNER_IMAGE \
+       $CONTAINER_ORDERLY_ROOT_PATH