From d98660ceb52cfc35c1c73b64275bac85a0caa361 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Mon, 7 Dec 2020 18:07:54 +0000 Subject: [PATCH 01/10] Add bundle pack/import for remotes --- R/remote.R | 24 ++++++++++++++++++++++++ R/remote_path.R | 11 +++++++++++ tests/testthat/test-remote-path.R | 18 ++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/R/remote.R b/R/remote.R index fe1a7fe60..581148be7 100644 --- a/R/remote.R +++ b/R/remote.R @@ -496,3 +496,27 @@ remote_report_update_metadata <- function(name, remote, config) { } data_frame(id = ids, path = file.path(path_cache, ids)) } + + +orderly_bundle_pack_remote <- function(name, parameters = NULL, + instance = NULL, + root = NULL, locate = TRUE, + remote = NULL, dest = tempdir()) { + remote <- get_remote(remote, orderly_config(root, locate)) + dir_create(dest) + path <- remote$bundle_pack(name, parameters = parameters, + instance = instance) + if (dest != tempdir()) { + fs::file_move(path, dest) + path <- file.path(dest, basename(path)) + } + + path +} + + +orderly_bundle_import_remote <- function(path, root = NULL, locate = TRUE, + remote = NULL) { + remote <- get_remote(remote, orderly_config(root, locate)) + remote$bundle_import(path) +} diff --git a/R/remote_path.R b/R/remote_path.R index da52dfac3..52ee13a5f 100644 --- a/R/remote_path.R +++ b/R/remote_path.R @@ -85,5 +85,16 @@ orderly_remote_path_ <- R6::R6Class( url_report = function(name, id) { file.path(self$config$root, name, id, fsep = "/") + }, + + bundle_pack = function(name, parameters = NULL, instance = NULL) { + res <- orderly_bundle_pack(tempdir(), name, parameters = parameters, + instance = instance, + root = self$config, locate = FALSE) + res$path + }, + + bundle_import = function(path) { + orderly_bundle_import(path, root = self$config, locate = FALSE) } )) diff --git a/tests/testthat/test-remote-path.R b/tests/testthat/test-remote-path.R index 2871b9bce..0ad34a5e5 100644 --- a/tests/testthat/test-remote-path.R +++ b/tests/testthat/test-remote-path.R @@ -300,3 +300,21 @@ test_that("pull dependencies that use a query", { orderly_list_archive(root), data_frame(name = "other", id = dat$ids[[2]])) }) + + +test_that("bundle pack and import", { + skip_on_cran_windows() + path1 <- prepare_orderly_example("minimal") + path2 <- prepare_orderly_example("minimal") + + remote <- orderly_remote_path(path1) + res <- orderly_bundle_pack_remote("example", remote = remote) + expect_equal(dirname(res), tempdir()) + expect_match(basename(res), "^[0-9]{8}-[0-9]{6}-[[:xdigit:]]{8}\\.zip$") + + ans <- orderly_bundle_run(res, echo = FALSE) + + orderly_bundle_import_remote(ans$path, remote = remote) + + expect_equal(remote$list_versions("example"), ans$id) +}) From 38178ce01fdaa3b92ce5b4c1c1791c5d9fb7d820 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 09:03:14 +0000 Subject: [PATCH 02/10] Test path more carefully --- tests/testthat/test-remote-path.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-remote-path.R b/tests/testthat/test-remote-path.R index 0ad34a5e5..7251b1f8c 100644 --- a/tests/testthat/test-remote-path.R +++ b/tests/testthat/test-remote-path.R @@ -309,7 +309,7 @@ test_that("bundle pack and import", { remote <- orderly_remote_path(path1) res <- orderly_bundle_pack_remote("example", remote = remote) - expect_equal(dirname(res), tempdir()) + expect_true(same_path(dirname(res), tempdir())) expect_match(basename(res), "^[0-9]{8}-[0-9]{6}-[[:xdigit:]]{8}\\.zip$") ans <- orderly_bundle_run(res, echo = FALSE) From 3fed20848b16dbfc50f52b68671e2b0c1746c532 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 10:09:21 +0000 Subject: [PATCH 03/10] Test directory handling behaviour --- tests/testthat/test-remote.R | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/testthat/test-remote.R b/tests/testthat/test-remote.R index d7fa75798..a51385c83 100644 --- a/tests/testthat/test-remote.R +++ b/tests/testthat/test-remote.R @@ -290,3 +290,34 @@ test_that("orderly run remote passes instance to run", { args <- mockery::mock_args(remote$run)[[2]] expect_equal(args$instance, "test") }) + + +test_that("orderly_bundle_(pack|import)_remote do not use root/locate", { + skip_on_cran_windows() + path <- prepare_orderly_example("minimal") + remote <- orderly_remote_path(path) + + temp <- tempfile() + on.exit(unlink(temp, recursive = TRUE)) + dir_create(temp) + + res <- withr::with_dir( + temp, + orderly_bundle_pack_remote("example", remote = remote, + root = stop("don't force me"), + locate = stop("don't force me"), + dest = ".")) + + expect_true(file.exists(file.path(temp, basename(res)))) + expect_equal(dirname(res), ".") + + ans <- orderly_bundle_run(file.path(temp, basename(res)), echo = FALSE) + + withr::with_dir( + temp, + orderly_bundle_import_remote(ans$path, remote = remote, + root = stop("don't force me"), + locate = stop("don't force me"))) + + expect_equal(remote$list_versions("example"), ans$id) +}) From fc6d05c8620a8b38cf53d05695c64b840448938d Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 10:22:01 +0000 Subject: [PATCH 04/10] Add basic docs --- DESCRIPTION | 2 +- NAMESPACE | 2 + R/bundle.R | 4 +- R/remote.R | 41 ++++++++++++++++ man/orderly_bundle_pack.Rd | 4 +- man/orderly_bundle_pack_remote.Rd | 81 +++++++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 man/orderly_bundle_pack_remote.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 5e00a4892..4c26cc66b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -44,7 +44,7 @@ Suggests: rmarkdown, testthat, vaultr (>= 1.0.4) -RoxygenNote: 7.1.0 +RoxygenNote: 7.1.1 VignetteBuilder: knitr Language: en-GB Remotes: vimc/vaultr diff --git a/NAMESPACE b/NAMESPACE index 5727338b9..05dd1c1e1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,8 +4,10 @@ S3method(format,orderly_deduplicate_info) S3method(print,orderly_deduplicate_info) export(orderly_batch) export(orderly_bundle_import) +export(orderly_bundle_import_remote) export(orderly_bundle_list) export(orderly_bundle_pack) +export(orderly_bundle_pack_remote) export(orderly_bundle_run) export(orderly_cleanup) export(orderly_commit) diff --git a/R/bundle.R b/R/bundle.R index ea58d90c6..316f47e66 100644 --- a/R/bundle.R +++ b/R/bundle.R @@ -18,8 +18,8 @@ ##' \item{\code{orderly_bundle_run}}{The path to the packed bundle (a zip ##' file created by \code{orderly_bundle_pack})} ##' -##' \item{\code{orderly_bundle_run}}{The path to unpack and the run -##' the bundle (a zip file created by \code{orderly_bundle_run}} +##' \item{\code{orderly_bundle_import}}{The path to unpack and import +##' (a zip file created by \code{orderly_bundle_run}} ##' ##' \item{\code{orderly_bundle_list}}{The path to a directory that might ##' contain either incomplete or complete bundles (created by either diff --git a/R/remote.R b/R/remote.R index 581148be7..d4c4edc96 100644 --- a/R/remote.R +++ b/R/remote.R @@ -498,6 +498,41 @@ remote_report_update_metadata <- function(name, remote, config) { } +##' Pack a bundle on a remote. This is like calling +##' \code{\link{orderly_bundle_pack}} on the remote and can be used to +##' extract a long-running report from a server to run (say) on a HPC +##' system. +##' +##' The workflow here will typically be: +##' +##' 1. Use \code{orderly_bundle_pack_remote()} to create a local +##' copy of a bundle, extracted from a remote. Typically this will +##' be run fro the system where the bundle will be run (an HPC +##' head-node or a other powerful computer). +##' +##' 2. Run the bundle using \code{\link{orderly_bundle_run}} +##' +##' 3. Re-import the completed bundle using +##' \code{orderly_bundle_import_remote} which sends the zip +##' file to the remote and adds it to the archive. +##' +##' Typically these commands will \emph{not} be run from the orderly +##' root. However, the \code{root} argument may still be used to find +##' your remote configuration. Alternatively, if your \code{remote} +##' argument is an orderly remote (e.g., +##' \code{\link{orderly_remote_path}}, or \code{orderlyweb}'s +##' \code{orderlyweb::orderlyweb_remote}) then the \code{root} and +##' \code{locate} arguments will be ignored and this command can be +##' run from anywhere. This is the recommended configuration for +##' running on a HPC system. +##' +##' @title Pack and import bundles with remotes +##' +##' @inheritParams orderly_bundle_pack +##' +##' @param remote The remote to pack the bundle from, or import to +##' +##' @export orderly_bundle_pack_remote <- function(name, parameters = NULL, instance = NULL, root = NULL, locate = TRUE, @@ -515,6 +550,12 @@ orderly_bundle_pack_remote <- function(name, parameters = NULL, } +##' @rdname orderly_bundle_pack_remote +##' +##' @param path The path to unpack and import +##' (a zip file created by \code{orderly_bundle_run}) +##' +##' @export orderly_bundle_import_remote <- function(path, root = NULL, locate = TRUE, remote = NULL) { remote <- get_remote(remote, orderly_config(root, locate)) diff --git a/man/orderly_bundle_pack.Rd b/man/orderly_bundle_pack.Rd index 5302bc82d..cffbe20ed 100644 --- a/man/orderly_bundle_pack.Rd +++ b/man/orderly_bundle_pack.Rd @@ -37,8 +37,8 @@ orderly_bundle_list(path) \item{\code{orderly_bundle_run}}{The path to the packed bundle (a zip file created by \code{orderly_bundle_pack})} -\item{\code{orderly_bundle_run}}{The path to unpack and the run - the bundle (a zip file created by \code{orderly_bundle_run}} +\item{\code{orderly_bundle_import}}{The path to unpack and import + (a zip file created by \code{orderly_bundle_run}} \item{\code{orderly_bundle_list}}{The path to a directory that might contain either incomplete or complete bundles (created by either diff --git a/man/orderly_bundle_pack_remote.Rd b/man/orderly_bundle_pack_remote.Rd new file mode 100644 index 000000000..3991e9c0e --- /dev/null +++ b/man/orderly_bundle_pack_remote.Rd @@ -0,0 +1,81 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/remote.R +\name{orderly_bundle_pack_remote} +\alias{orderly_bundle_pack_remote} +\alias{orderly_bundle_import_remote} +\title{Pack and import bundles with remotes} +\usage{ +orderly_bundle_pack_remote( + name, + parameters = NULL, + instance = NULL, + root = NULL, + locate = TRUE, + remote = NULL, + dest = tempdir() +) + +orderly_bundle_import_remote(path, root = NULL, locate = TRUE, remote = NULL) +} +\arguments{ +\item{name}{Name of the report to pack (see +\code{\link{orderly_list}}). A leading \code{src/} will be +removed if provided, allowing easier use of autocomplete.} + +\item{parameters}{Parameters passed to the report. A named list of +parameters declared in the \code{orderly.yml}. Each parameter +must be a scalar character, numeric, integer or logical.} + +\item{instance}{Select instance of the source database to be used, +where multiple instances are configured. Use a single +\emph{unnamed} character string to indicate an instance to +match. If given, then this name must be present in all +databases where instances are listed in +\code{orderly_config.yml}, and will be ignored by all database +where instances are not given. See the "orderly" vignette for +further information.} + +\item{root}{The path to an orderly root directory, or \code{NULL} +(the default) to search for one from the current working +directory if \code{locate} is \code{TRUE}.} + +\item{locate}{Logical, indicating if the configuration should be +searched for. If \code{TRUE} and \code{config} is not given, +then orderly looks in the working directory and up through its +parents until it finds an \code{orderly_config.yml} file.} + +\item{remote}{The remote to pack the bundle from, or import to} + +\item{path}{The path to unpack and import +(a zip file created by \code{orderly_bundle_run})} +} +\description{ +Pack a bundle on a remote. This is like calling +\code{\link{orderly_bundle_pack}} on the remote and can be used to +extract a long-running report from a server to run (say) on a HPC +system. +} +\details{ +The workflow here will typically be: + +1. Use \code{orderly_bundle_pack_remote()} to create a local + copy of a bundle, extracted from a remote. Typically this will + be run fro the system where the bundle will be run (an HPC + head-node or a other powerful computer). + +2. Run the bundle using \code{\link{orderly_bundle_run}} + +3. Re-import the completed bundle using + \code{orderly_bundle_import_remote} which sends the zip + file to the remote and adds it to the archive. + +Typically these commands will \emph{not} be run from the orderly +root. However, the \code{root} argument may still be used to find +your remote configuration. Alternatively, if your \code{remote} +argument is an orderly remote (e.g., +\code{\link{orderly_remote_path}}, or \code{orderlyweb}'s +\code{orderlyweb::orderlyweb_remote}) then the \code{root} and +\code{locate} arguments will be ignored and this command can be +run from anywhere. This is the recommended configuration for +running on a HPC system. +} From 11afd0d609d4c017949938a05f79201a45e1547f Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 10:22:14 +0000 Subject: [PATCH 05/10] Bump version and add news --- DESCRIPTION | 2 +- NEWS.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 4c26cc66b..3242b0cac 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: orderly Title: Lightweight Reproducible Reporting -Version: 1.2.18 +Version: 1.2.19 Description: Order, create and store reports from R. By defining a lightweight interface around the inputs and outputs of an analysis, a lot of the repetitive work for reproducible research diff --git a/NEWS.md b/NEWS.md index 325dec9aa..ee0527b7a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# orderly 1.2.19 + +* New functions `orderly_bundle_pack_remote` and `orderly_bundle_import_remote` which create a bundle from a remote and return the completed bundle back to the remote (VIMC-4457) + # orderly 1.2.18 * Allow `orderly_bundle_import` to accept a filename that has been renamed from `.zip`. While this is not generally desireable, it may be needed for some workflows (VIMC-4382) From 7486c6ac90d16dcabbde5fc7917174c1f8642cd3 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 10:22:41 +0000 Subject: [PATCH 06/10] Add release workflow --- .github/workflows/make-release.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/make-release.yaml diff --git a/.github/workflows/make-release.yaml b/.github/workflows/make-release.yaml new file mode 100644 index 000000000..1fc3753be --- /dev/null +++ b/.github/workflows/make-release.yaml @@ -0,0 +1,29 @@ +on: + push: + branches: + - master + +name: make-release + +jobs: + create-release: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Extract version + run: | + echo "PACKAGE_VERSION=$(grep '^Version' DESCRIPTION | sed 's/.*: *//')" >> $GITHUB_ENV + echo "PACKAGE_NAME=$(grep '^Package' DESCRIPTION | sed 's/.*: *//')" >> $GITHUB_ENV + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ env.PACKAGE_VERSION }} + release_name: Release ${{ env.PACKAGE_NAME }} ${{ env.PACKAGE_VERSION }} + draft: false + prerelease: false From 9ba6823d21641841922e40d6a1ccbcd08f184a6f Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 10:34:54 +0000 Subject: [PATCH 07/10] Add missing arg to docs --- R/remote.R | 6 +++++- man/orderly_bundle_pack_remote.Rd | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/R/remote.R b/R/remote.R index d4c4edc96..fad307f26 100644 --- a/R/remote.R +++ b/R/remote.R @@ -530,7 +530,11 @@ remote_report_update_metadata <- function(name, remote, config) { ##' ##' @inheritParams orderly_bundle_pack ##' -##' @param remote The remote to pack the bundle from, or import to +##' @param remote The remote to pack the bundle from, or import into +##' +##' @param dest Optional path to write bundle to (a directory +##' name). By default we use the temporary directory and return the +##' full path to the created file. ##' ##' @export orderly_bundle_pack_remote <- function(name, parameters = NULL, diff --git a/man/orderly_bundle_pack_remote.Rd b/man/orderly_bundle_pack_remote.Rd index 3991e9c0e..bca90adb9 100644 --- a/man/orderly_bundle_pack_remote.Rd +++ b/man/orderly_bundle_pack_remote.Rd @@ -44,7 +44,11 @@ searched for. If \code{TRUE} and \code{config} is not given, then orderly looks in the working directory and up through its parents until it finds an \code{orderly_config.yml} file.} -\item{remote}{The remote to pack the bundle from, or import to} +\item{remote}{The remote to pack the bundle from, or import into} + +\item{dest}{Optional path to write bundle to (a directory +name). By default we use the temporary directory and return the +full path to the created file.} \item{path}{The path to unpack and import (a zip file created by \code{orderly_bundle_run})} From d8d86432f32db28fa79a9ec786f2f392b0dc618b Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 14:46:32 +0000 Subject: [PATCH 08/10] Apply suggestions from code review Co-authored-by: Mark Woodbridge <1724545+mwoodbri@users.noreply.github.com> --- R/bundle.R | 2 +- R/remote.R | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/bundle.R b/R/bundle.R index 316f47e66..e9dff6734 100644 --- a/R/bundle.R +++ b/R/bundle.R @@ -19,7 +19,7 @@ ##' file created by \code{orderly_bundle_pack})} ##' ##' \item{\code{orderly_bundle_import}}{The path to unpack and import -##' (a zip file created by \code{orderly_bundle_run}} +##' (a zip file created by \code{orderly_bundle_run})} ##' ##' \item{\code{orderly_bundle_list}}{The path to a directory that might ##' contain either incomplete or complete bundles (created by either diff --git a/R/remote.R b/R/remote.R index fad307f26..0defc9ba0 100644 --- a/R/remote.R +++ b/R/remote.R @@ -500,15 +500,15 @@ remote_report_update_metadata <- function(name, remote, config) { ##' Pack a bundle on a remote. This is like calling ##' \code{\link{orderly_bundle_pack}} on the remote and can be used to -##' extract a long-running report from a server to run (say) on a HPC +##' extract a long-running report from a server to run (say) on an HPC ##' system. ##' ##' The workflow here will typically be: ##' ##' 1. Use \code{orderly_bundle_pack_remote()} to create a local ##' copy of a bundle, extracted from a remote. Typically this will -##' be run fro the system where the bundle will be run (an HPC -##' head-node or a other powerful computer). +##' be run from the system where the bundle will be run (an HPC +##' head-node or another powerful computer). ##' ##' 2. Run the bundle using \code{\link{orderly_bundle_run}} ##' From 2f05500bb3bef400f8269d54fd8f38f677e39bac Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 14:47:23 +0000 Subject: [PATCH 09/10] Redocument --- man/orderly_bundle_pack.Rd | 2 +- man/orderly_bundle_pack_remote.Rd | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/man/orderly_bundle_pack.Rd b/man/orderly_bundle_pack.Rd index cffbe20ed..d091113a9 100644 --- a/man/orderly_bundle_pack.Rd +++ b/man/orderly_bundle_pack.Rd @@ -38,7 +38,7 @@ orderly_bundle_list(path) file created by \code{orderly_bundle_pack})} \item{\code{orderly_bundle_import}}{The path to unpack and import - (a zip file created by \code{orderly_bundle_run}} + (a zip file created by \code{orderly_bundle_run})} \item{\code{orderly_bundle_list}}{The path to a directory that might contain either incomplete or complete bundles (created by either diff --git a/man/orderly_bundle_pack_remote.Rd b/man/orderly_bundle_pack_remote.Rd index bca90adb9..b67edfd16 100644 --- a/man/orderly_bundle_pack_remote.Rd +++ b/man/orderly_bundle_pack_remote.Rd @@ -56,7 +56,7 @@ full path to the created file.} \description{ Pack a bundle on a remote. This is like calling \code{\link{orderly_bundle_pack}} on the remote and can be used to -extract a long-running report from a server to run (say) on a HPC +extract a long-running report from a server to run (say) on an HPC system. } \details{ @@ -64,8 +64,8 @@ The workflow here will typically be: 1. Use \code{orderly_bundle_pack_remote()} to create a local copy of a bundle, extracted from a remote. Typically this will - be run fro the system where the bundle will be run (an HPC - head-node or a other powerful computer). + be run from the system where the bundle will be run (an HPC + head-node or another powerful computer). 2. Run the bundle using \code{\link{orderly_bundle_run}} From 2f1cad8b50cbf9e86f060a504fd0c5bc512406d7 Mon Sep 17 00:00:00 2001 From: Rich FitzJohn Date: Tue, 8 Dec 2020 14:48:08 +0000 Subject: [PATCH 10/10] Eliminate unused variable in test --- tests/testthat/test-remote-path.R | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test-remote-path.R b/tests/testthat/test-remote-path.R index 7251b1f8c..94195c3a5 100644 --- a/tests/testthat/test-remote-path.R +++ b/tests/testthat/test-remote-path.R @@ -304,10 +304,9 @@ test_that("pull dependencies that use a query", { test_that("bundle pack and import", { skip_on_cran_windows() - path1 <- prepare_orderly_example("minimal") - path2 <- prepare_orderly_example("minimal") + path <- prepare_orderly_example("minimal") - remote <- orderly_remote_path(path1) + remote <- orderly_remote_path(path) res <- orderly_bundle_pack_remote("example", remote = remote) expect_true(same_path(dirname(res), tempdir())) expect_match(basename(res), "^[0-9]{8}-[0-9]{6}-[[:xdigit:]]{8}\\.zip$")