From 6eb7c9dc16179d9a0991dd36349919f05d110a30 Mon Sep 17 00:00:00 2001 From: Yuxin Shi Date: Tue, 17 Jan 2023 14:55:15 -0800 Subject: [PATCH 1/2] wip: use nxf_dx_upload and simplify subjob inputs --- .../templates/nextflow/src/nextflow.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh b/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh index 08b07bd3c1..abccb3eb1a 100644 --- a/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh +++ b/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh @@ -85,9 +85,7 @@ on_exit() { # wrap cache folder and upload cache.tar if [[ -n "$(ls -A .nextflow)" ]]; then tar -cf cache.tar .nextflow - - CACHE_ID=$(dx upload "cache.tar" --path "$DX_CACHEDIR/$NXF_UUID/cache.tar" --no-progress --brief --wait -p -r) && - echo "Upload cache of current session as file: $CACHE_ID" && + nxf_dx_upload "cache.tar" "$DX_CACHEDIR/$NXF_UUID" && rm -f cache.tar || echo "Failed to upload cache of current session $NXF_UUID" else @@ -403,10 +401,17 @@ main() { exit $ret } +nxf_dx_upload() { + local name=$1 + local target=${2%/} + escape_name="${name//:/\\:}" + dx upload "$name" --path "$target/$escape_name" --no-progress --brief --wait -p -r | (echo -n "Upload $name to $target/$escape_name: " && cat) || true +} + nf_task_exit() { ret=$? if [ -f .command.log ]; then - dx upload .command.log --path "${cmd_log_file}" --brief --wait --no-progress || true + nxf_dx_upload .command.log "$TASK_WORKDIR" else >&2 echo "Missing Nextflow .command.log file" fi @@ -433,8 +438,10 @@ nf_task_entry() { fi # capture the exit code trap nf_task_exit EXIT + + export -f nxf_dx_upload # remove the line in .command.run to disable printing env vars if debugging is on - dx cat "${cmd_launcher_file}" | sed 's/\[\[ $NXF_DEBUG > 0 ]] && nxf_env//' > .command.run + dx cat "${TASK_WORKDIR}/.command.run" | sed 's/\[\[ $NXF_DEBUG > 0 ]] && nxf_env//' > .command.run set +e # enable debugging mode [[ $NXF_DEBUG ]] && set -x From 62837a8642a46cc71228bfa2045623f336d5ecd2 Mon Sep 17 00:00:00 2001 From: Yuxin Shi Date: Wed, 18 Jan 2023 17:02:06 -0800 Subject: [PATCH 2/2] revert export nxf_dx_upload --- src/python/dxpy/templating/templates/nextflow/src/nextflow.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh b/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh index abccb3eb1a..58f338d866 100644 --- a/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh +++ b/src/python/dxpy/templating/templates/nextflow/src/nextflow.sh @@ -439,7 +439,6 @@ nf_task_entry() { # capture the exit code trap nf_task_exit EXIT - export -f nxf_dx_upload # remove the line in .command.run to disable printing env vars if debugging is on dx cat "${TASK_WORKDIR}/.command.run" | sed 's/\[\[ $NXF_DEBUG > 0 ]] && nxf_env//' > .command.run set +e