Merge pull request #258 from vimc/vimc-4457

vimc-4457: Add bundle pack/import for remotes

.github/workflows/make-release.yaml

@@ -0,0 +1,29 @@
+on:
+  push:
+    branches:
+      - master
+
+name: make-release
+
+jobs:
+  create-release:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Extract version
+        run: |
+          echo "PACKAGE_VERSION=$(grep '^Version' DESCRIPTION  | sed 's/.*: *//')" >> $GITHUB_ENV
+          echo "PACKAGE_NAME=$(grep '^Package' DESCRIPTION  | sed 's/.*: *//')" >> $GITHUB_ENV
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: v${{ env.PACKAGE_VERSION }}
+          release_name: Release ${{ env.PACKAGE_NAME }} ${{ env.PACKAGE_VERSION }}
+          draft: false
+          prerelease: false

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: orderly
 Title: Lightweight Reproducible Reporting
-Version: 1.2.18
+Version: 1.2.19
 Description: Order, create and store reports from R.  By defining a
     lightweight interface around the inputs and outputs of an
     analysis, a lot of the repetitive work for reproducible research
@@ -44,7 +44,7 @@ Suggests:
     rmarkdown,
     testthat,
     vaultr (>= 1.0.4)
-RoxygenNote: 7.1.0
+RoxygenNote: 7.1.1
 VignetteBuilder: knitr
 Language: en-GB
 Remotes: vimc/vaultr

NAMESPACE

@@ -4,8 +4,10 @@ S3method(format,orderly_deduplicate_info)
 S3method(print,orderly_deduplicate_info)
 export(orderly_batch)
 export(orderly_bundle_import)
+export(orderly_bundle_import_remote)
 export(orderly_bundle_list)
 export(orderly_bundle_pack)
+export(orderly_bundle_pack_remote)
 export(orderly_bundle_run)
 export(orderly_cleanup)
 export(orderly_commit)

NEWS.md

@@ -1,3 +1,7 @@
+# orderly 1.2.19
+
+* New functions `orderly_bundle_pack_remote` and `orderly_bundle_import_remote` which create a bundle from a remote and return the completed bundle back to the remote (VIMC-4457)
+
 # orderly 1.2.18
 
 * Allow `orderly_bundle_import` to accept a filename that has been renamed from `<id>.zip`. While this is not generally desireable, it may be needed for some workflows (VIMC-4382)

R/bundle.R

@@ -18,8 +18,8 @@
 ##' \item{\code{orderly_bundle_run}}{The path to the packed bundle (a zip
 ##'   file created by \code{orderly_bundle_pack})}
 ##'
-##' \item{\code{orderly_bundle_run}}{The path to unpack and the run
-##'   the bundle (a zip file created by \code{orderly_bundle_run}}
+##' \item{\code{orderly_bundle_import}}{The path to unpack and import
+##'   (a zip file created by \code{orderly_bundle_run})}
 ##'
 ##' \item{\code{orderly_bundle_list}}{The path to a directory that might
 ##'   contain either incomplete or complete bundles (created by either

R/remote.R

@@ -496,3 +496,72 @@ remote_report_update_metadata <- function(name, remote, config) {
   }
   data_frame(id = ids, path = file.path(path_cache, ids))
 }
+
+
+##' Pack a bundle on a remote. This is like calling
+##' \code{\link{orderly_bundle_pack}} on the remote and can be used to
+##' extract a long-running report from a server to run (say) on an HPC
+##' system.
+##'
+##' The workflow here will typically be:
+##'
+##' 1. Use \code{orderly_bundle_pack_remote()} to create a local
+##'    copy of a bundle, extracted from a remote. Typically this will
+##'    be run from the system where the bundle will be run (an HPC
+##'    head-node or another powerful computer).
+##'
+##' 2. Run the bundle using \code{\link{orderly_bundle_run}}
+##'
+##' 3. Re-import the completed bundle using
+##'   \code{orderly_bundle_import_remote} which sends the zip
+##'   file to the remote and adds it to the archive.
+##'
+##' Typically these commands will \emph{not} be run from the orderly
+##' root. However, the \code{root} argument may still be used to find
+##' your remote configuration. Alternatively, if your \code{remote}
+##' argument is an orderly remote (e.g.,
+##' \code{\link{orderly_remote_path}}, or \code{orderlyweb}'s
+##' \code{orderlyweb::orderlyweb_remote}) then the \code{root} and
+##' \code{locate} arguments will be ignored and this command can be
+##' run from anywhere. This is the recommended configuration for
+##' running on a HPC system.
+##'
+##' @title Pack and import bundles with remotes
+##'
+##' @inheritParams orderly_bundle_pack
+##'
+##' @param remote The remote to pack the bundle from, or import into
+##'
+##' @param dest Optional path to write bundle to (a directory
+##'   name). By default we use the temporary directory and return the
+##'   full path to the created file.
+##'
+##' @export
+orderly_bundle_pack_remote <- function(name, parameters = NULL,
+                                       instance = NULL,
+                                       root = NULL, locate = TRUE,
+                                       remote = NULL, dest = tempdir()) {
+  remote <- get_remote(remote, orderly_config(root, locate))
+  dir_create(dest)
+  path <- remote$bundle_pack(name, parameters = parameters,
+                             instance = instance)
+  if (dest != tempdir()) {
+    fs::file_move(path, dest)
+    path <- file.path(dest, basename(path))
+  }
+
+  path
+}
+
+
+##' @rdname orderly_bundle_pack_remote
+##'
+##' @param path The path to unpack and import
+##'   (a zip file created by \code{orderly_bundle_run})
+##'
+##' @export
+orderly_bundle_import_remote <- function(path, root = NULL, locate = TRUE,
+                                         remote = NULL) {
+  remote <- get_remote(remote, orderly_config(root, locate))
+  remote$bundle_import(path)
+}

R/remote_path.R

@@ -85,5 +85,16 @@ orderly_remote_path_ <- R6::R6Class(
 
     url_report = function(name, id) {
       file.path(self$config$root, name, id, fsep = "/")
+    },
+
+    bundle_pack = function(name, parameters = NULL, instance = NULL) {
+      res <- orderly_bundle_pack(tempdir(), name, parameters = parameters,
+                                 instance = instance,
+                                 root = self$config, locate = FALSE)
+      res$path
+    },
+
+    bundle_import = function(path) {
+      orderly_bundle_import(path, root = self$config, locate = FALSE)
     }
   ))

tests/testthat/test-remote-path.R

@@ -300,3 +300,20 @@ test_that("pull dependencies that use a query", {
     orderly_list_archive(root),
     data_frame(name = "other", id = dat$ids[[2]]))
 })
+
+
+test_that("bundle pack and import", {
+  skip_on_cran_windows()
+  path <- prepare_orderly_example("minimal")
+
+  remote <- orderly_remote_path(path)
+  res <- orderly_bundle_pack_remote("example", remote = remote)
+  expect_true(same_path(dirname(res), tempdir()))
+  expect_match(basename(res), "^[0-9]{8}-[0-9]{6}-[[:xdigit:]]{8}\\.zip$")
+
+  ans <- orderly_bundle_run(res, echo = FALSE)
+
+  orderly_bundle_import_remote(ans$path, remote = remote)
+
+  expect_equal(remote$list_versions("example"), ans$id)
+})

tests/testthat/test-remote.R

@@ -290,3 +290,34 @@ test_that("orderly run remote passes instance to run", {
   args <- mockery::mock_args(remote$run)[[2]]
   expect_equal(args$instance, "test")
 })
+
+
+test_that("orderly_bundle_(pack|import)_remote do not use root/locate", {
+  skip_on_cran_windows()
+  path <- prepare_orderly_example("minimal")
+  remote <- orderly_remote_path(path)
+
+  temp <- tempfile()
+  on.exit(unlink(temp, recursive = TRUE))
+  dir_create(temp)
+
+  res <- withr::with_dir(
+    temp,
+    orderly_bundle_pack_remote("example", remote = remote,
+                               root = stop("don't force me"),
+                               locate = stop("don't force me"),
+                               dest = "."))
+
+  expect_true(file.exists(file.path(temp, basename(res))))
+  expect_equal(dirname(res), ".")
+
+  ans <- orderly_bundle_run(file.path(temp, basename(res)), echo = FALSE)
+
+  withr::with_dir(
+    temp,
+    orderly_bundle_import_remote(ans$path, remote = remote,
+                                 root = stop("don't force me"),
+                                 locate = stop("don't force me")))
+
+  expect_equal(remote$list_versions("example"), ans$id)
+})