From ce5c0301ccd4be6c4e8ec6e5432103eadf0c0183 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Wed, 13 Nov 2024 16:03:57 +0000 Subject: [PATCH 1/4] Use new rrq offload configuration API. rrq 0.7.20 has changed a bit how the offload gets configured. Instead of being a global property that must be configured ahead of time, the path to the offload directory is now set per-controller/worker. The offload threshold size is similar, although for workers it is set through the worker configuration. --- DESCRIPTION | 4 ++-- R/main.R | 9 +-------- R/queue.R | 12 +++++++++--- docker/Dockerfile | 1 - docker/daedalus.api.configure | 2 -- docker/run_containers | 9 +-------- 6 files changed, 13 insertions(+), 24 deletions(-) delete mode 100755 docker/daedalus.api.configure diff --git a/DESCRIPTION b/DESCRIPTION index d176f31..b5a0877 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -21,7 +21,7 @@ Imports: porcelain, R6, redux, - rrq, + rrq (>= 0.7.20), stringr, tidyr Suggests: @@ -33,7 +33,7 @@ Suggests: withr Remotes: jameel-institute/daedalus, - mrc-ide/rrq, + mrc-ide/rrq@mrc-5889, reside-ic/porcelain Config/testthat/edition: 3 Encoding: UTF-8 diff --git a/R/main.R b/R/main.R index 892cb5f..4e2e2be 100644 --- a/R/main.R +++ b/R/main.R @@ -25,15 +25,8 @@ main <- function(args = commandArgs(TRUE)) { main_worker <- function() { worker <- rrq::rrq_worker$new( get_queue_id(), + offload_path = get_results_dir(), con = get_redis_connection() ) worker$loop() } - -main_configure_queue <- function() { - queue_id <- get_queue_id() - rrq::rrq_configure(queue_id, - store_max_size = 1000L, - offload_path = get_results_dir(), - con = get_redis_connection()) -} diff --git a/R/queue.R b/R/queue.R index c8d42db..e0bcbf3 100644 --- a/R/queue.R +++ b/R/queue.R @@ -18,13 +18,19 @@ Queue <- R6::R6Class("Queue", # nolint # Configure rrq to store data > 1KB to disk queue_id <- get_queue_id() - # TODO: Check that the queue is configured, when rrq api is available # nolint # Create queue - self$controller <- rrq::rrq_controller(queue_id, con = con) + self$controller <- rrq::rrq_controller( + queue_id, + offload_threshold_size = 1000L, + offload_path = results_dir, + con = con) + dir.create(logs_dir, showWarnings = FALSE) dir.create(results_dir, showWarnings = FALSE) - worker_config <- rrq::rrq_worker_config(logdir = logs_dir) + worker_config <- rrq::rrq_worker_config( + offload_threshold_size = 1000L, + logdir = logs_dir) rrq::rrq_worker_config_save("localhost", worker_config, controller = self$controller) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5288d3c..120718e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,7 +40,6 @@ COPY . /src RUN R CMD INSTALL --install-tests /src && \ cp /src/docker/daedalus.api /usr/local/bin && \ cp /src/docker/daedalus.api.worker /usr/local/bin && \ - cp /src/docker/daedalus.api.configure /usr/local/bin && \ rm -rf /src # ENTRYPOINT for api is "/usr/local/bin/daedalus.api" diff --git a/docker/daedalus.api.configure b/docker/daedalus.api.configure deleted file mode 100755 index bc71941..0000000 --- a/docker/daedalus.api.configure +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env Rscript -daedalus.api:::main_configure_queue() \ No newline at end of file diff --git a/docker/run_containers b/docker/run_containers index 2fcac77..17d4cac 100755 --- a/docker/run_containers +++ b/docker/run_containers @@ -14,13 +14,6 @@ docker run --rm -d \ --network=$NAME_NETWORK \ redis -docker run --rm \ - --network=$NAME_NETWORK \ - --env=DAEDALUS_QUEUE_ID=$QUEUE_ID \ - --env=REDIS_CONTAINER_NAME=$NAME_REDIS \ - --entrypoint="/usr/local/bin/daedalus.api.configure" \ - $TAG_SHA - docker run --rm -d \ -p 8001:8001 \ --name=$NAME_SERVER \ @@ -38,4 +31,4 @@ docker run --rm -d \ --env=DAEDALUS_QUEUE_ID=$QUEUE_ID \ --env=REDIS_CONTAINER_NAME=$NAME_REDIS\ --entrypoint="/usr/local/bin/daedalus.api.worker" \ - $TAG_SHA \ No newline at end of file + $TAG_SHA From 3e6ad0b8791227c6649dd76895321541f5dadf09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Wed, 13 Nov 2024 19:03:00 +0000 Subject: [PATCH 2/4] Fix tests --- DESCRIPTION | 2 +- R/queue.R | 11 ++++++----- tests/testthat/helper-daedalus-api.R | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index b5a0877..aa256e7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -21,7 +21,7 @@ Imports: porcelain, R6, redux, - rrq (>= 0.7.20), + rrq (>= 0.7.21), stringr, tidyr Suggests: diff --git a/R/queue.R b/R/queue.R index e0bcbf3..cec6998 100644 --- a/R/queue.R +++ b/R/queue.R @@ -13,23 +13,24 @@ Queue <- R6::R6Class("Queue", # nolint logs_dir <- get_logs_dir() results_dir <- get_results_dir() + # Configure rrq to store data > 1KB to disk + offload_threshold_size <- 1000L + # Connect to Redis con <- get_redis_connection() - # Configure rrq to store data > 1KB to disk - queue_id <- get_queue_id() - # Create queue + queue_id <- get_queue_id() self$controller <- rrq::rrq_controller( queue_id, - offload_threshold_size = 1000L, + offload_threshold_size = offload_threshold_size, offload_path = results_dir, con = con) dir.create(logs_dir, showWarnings = FALSE) dir.create(results_dir, showWarnings = FALSE) worker_config <- rrq::rrq_worker_config( - offload_threshold_size = 1000L, + offload_threshold_size = offload_threshold_size, logdir = logs_dir) rrq::rrq_worker_config_save("localhost", worker_config, diff --git a/tests/testthat/helper-daedalus-api.R b/tests/testthat/helper-daedalus-api.R index ed54da8..888513d 100644 --- a/tests/testthat/helper-daedalus-api.R +++ b/tests/testthat/helper-daedalus-api.R @@ -12,7 +12,8 @@ check_for_redis <- function() { start_test_queue_with_worker <- function() { queue <- Queue$new() # nolint - rrq::rrq_worker_spawn(1L, controller = queue$controller) + rrq::rrq_worker_spawn(1L, controller = queue$controller, + offload_path = get_results_dir()) queue } From 75d2e98970781bcae5215058a42e3f3ae0d296fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Mon, 18 Nov 2024 13:17:27 +0000 Subject: [PATCH 3/4] Remove rrq branch ref --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index aa256e7..ea45a87 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -33,7 +33,7 @@ Suggests: withr Remotes: jameel-institute/daedalus, - mrc-ide/rrq@mrc-5889, + mrc-ide/rrq, reside-ic/porcelain Config/testthat/edition: 3 Encoding: UTF-8 From 872bc1f5405ec9121619cd9034610355228eab5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Li=C3=A9tar?= Date: Wed, 20 Nov 2024 15:12:44 +0000 Subject: [PATCH 4/4] Bump version number --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ea45a87..156f1a1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: daedalus.api Title: Serve the 'daedalus' model via an API -Version: 0.1.0 +Version: 0.1.1 Authors@R: c( person("Pratik", "Gupte", , "p.gupte24@imperial.ac.uk", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-5294-7819")),