From 2a3da07f33b21c909bba068f699bed21d1a58bba Mon Sep 17 00:00:00 2001 From: Sage Wright Date: Wed, 16 Oct 2024 19:29:29 +0000 Subject: [PATCH 1/3] tada --- tasks/utilities/data_export/task_broad_terra_tools.wdl | 3 ++- tasks/utilities/data_export/task_download_terra_table.wdl | 4 +++- tasks/utilities/data_export/task_export_two_tsvs.wdl | 1 + tasks/utilities/data_import/task_create_terra_table.wdl | 4 ++++ tasks/utilities/file_handling/task_transfer_files.wdl | 2 ++ tasks/utilities/submission/task_submission.wdl | 2 ++ 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tasks/utilities/data_export/task_broad_terra_tools.wdl b/tasks/utilities/data_export/task_broad_terra_tools.wdl index 3a3fba0fd..5292ccbc9 100644 --- a/tasks/utilities/data_export/task_broad_terra_tools.wdl +++ b/tasks/utilities/data_export/task_broad_terra_tools.wdl @@ -390,7 +390,8 @@ task export_taxon_tables { volatile: true } command <<< - + set -euo pipefail + # capture taxon and corresponding table names from input taxon_tables taxon_array=($(cut -f1 ~{taxon_tables} | tail +2)) echo "Taxon array: ${taxon_array[*]}" diff --git a/tasks/utilities/data_export/task_download_terra_table.wdl b/tasks/utilities/data_export/task_download_terra_table.wdl index 3068de730..49d71c0be 100644 --- a/tasks/utilities/data_export/task_download_terra_table.wdl +++ b/tasks/utilities/data_export/task_download_terra_table.wdl @@ -12,11 +12,13 @@ task download_terra_table { String terra_workspace_name String terra_project_name Int disk_size = 10 - Int memory = 1 + Int memory = 2 Int cpu = 1 String docker = "us-docker.pkg.dev/general-theiagen/theiagen/terra-tools:2023-06-21" } command <<< + set -euo pipefail + python3 /scripts/export_large_tsv/export_large_tsv.py --project ~{terra_project_name} --workspace ~{terra_workspace_name} --entity_type ~{terra_table_name} --tsv_filename "~{terra_table_name}.tsv" >>> output { diff --git a/tasks/utilities/data_export/task_export_two_tsvs.wdl b/tasks/utilities/data_export/task_export_two_tsvs.wdl index d3707441f..4410e29a8 100644 --- a/tasks/utilities/data_export/task_export_two_tsvs.wdl +++ b/tasks/utilities/data_export/task_export_two_tsvs.wdl @@ -18,6 +18,7 @@ task export_two_tsvs { volatile: true } command <<< + set -euo pipefail python3 /scripts/export_large_tsv/export_large_tsv.py --project ~{terra_project1} --workspace ~{terra_workspace1} --entity_type ~{datatable1} --tsv_filename "~{datatable1}_table1.tsv" # check if second project is provided; if not, use first diff --git a/tasks/utilities/data_import/task_create_terra_table.wdl b/tasks/utilities/data_import/task_create_terra_table.wdl index 638052ab0..22f95453a 100644 --- a/tasks/utilities/data_import/task_create_terra_table.wdl +++ b/tasks/utilities/data_import/task_create_terra_table.wdl @@ -146,6 +146,10 @@ task create_terra_table { done >> output { diff --git a/tasks/utilities/file_handling/task_transfer_files.wdl b/tasks/utilities/file_handling/task_transfer_files.wdl index 28cfbebb9..1115df119 100644 --- a/tasks/utilities/file_handling/task_transfer_files.wdl +++ b/tasks/utilities/file_handling/task_transfer_files.wdl @@ -14,6 +14,8 @@ task transfer_files { volatile: true } command <<< + set -euo pipefail + file_path_array="~{sep=' ' files_to_transfer}" gsutil -m cp -n ${file_path_array[@]} ~{target_bucket} diff --git a/tasks/utilities/submission/task_submission.wdl b/tasks/utilities/submission/task_submission.wdl index 694b4f0e8..ab384c86b 100644 --- a/tasks/utilities/submission/task_submission.wdl +++ b/tasks/utilities/submission/task_submission.wdl @@ -23,6 +23,8 @@ task prune_table { volatile: true } command <<< + set -euo pipefail + # when running on terra, comment out all input_table mentions python3 /scripts/export_large_tsv/export_large_tsv.py --project "~{project_name}" --workspace "~{workspace_name}" --entity_type ~{table_name} --tsv_filename ~{table_name}-data.tsv From c425b04dedbe64b51e1196e211fc93a02d553fe5 Mon Sep 17 00:00:00 2001 From: Sage Wright Date: Thu, 31 Oct 2024 15:40:03 +0000 Subject: [PATCH 2/3] two more --- tasks/utilities/data_handling/task_summarize_data.wdl | 2 ++ tasks/utilities/data_handling/task_theiacov_fasta_batch.wdl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tasks/utilities/data_handling/task_summarize_data.wdl b/tasks/utilities/data_handling/task_summarize_data.wdl index 40586fbf3..5e5f64468 100644 --- a/tasks/utilities/data_handling/task_summarize_data.wdl +++ b/tasks/utilities/data_handling/task_summarize_data.wdl @@ -23,6 +23,8 @@ task summarize_data { volatile: true } command <<< + set -euo pipefail + # when running on terra, comment out all input_table mentions python3 /scripts/export_large_tsv/export_large_tsv.py --project "~{terra_project}" --workspace "~{terra_workspace}" --entity_type ~{terra_table} --tsv_filename ~{terra_table}-data.tsv diff --git a/tasks/utilities/data_handling/task_theiacov_fasta_batch.wdl b/tasks/utilities/data_handling/task_theiacov_fasta_batch.wdl index 5ab9247ad..4eb101b2e 100644 --- a/tasks/utilities/data_handling/task_theiacov_fasta_batch.wdl +++ b/tasks/utilities/data_handling/task_theiacov_fasta_batch.wdl @@ -28,6 +28,8 @@ task sm_theiacov_fasta_wrangling { # the sm stands for supermassive Int memory = 4 } command <<< + set -euo pipefail + # check if nextclade json file exists if [ -f ~{nextclade_json} ]; then # this line splits into individual json files From 7894c6484c970094e7c25dc7e01513e159eead13 Mon Sep 17 00:00:00 2001 From: Sage Wright Date: Fri, 8 Nov 2024 16:05:58 +0000 Subject: [PATCH 3/3] update --- tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml | 2 +- tests/workflows/theiaprok/test_wf_theiaprok_illumina_se.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml b/tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml index 71f5bd4a2..ce057f3ed 100644 --- a/tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml +++ b/tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml @@ -629,7 +629,7 @@ - path: miniwdl_run/wdl/tasks/taxon_id/contamination/task_midas.wdl md5sum: 64caaaff5910ac0036e2659434500962 - path: miniwdl_run/wdl/tasks/utilities/data_export/task_broad_terra_tools.wdl - md5sum: 4d69a6539b68503af9f3f1c2787ff920 + md5sum: 850ad97598aca5c28eb36e6a5c13c2fc - path: miniwdl_run/wdl/workflows/theiaprok/wf_theiaprok_illumina_pe.wdl md5sum: 3cb5c86b15e931b0c0b98ed784386438 - path: miniwdl_run/wdl/workflows/utilities/wf_merlin_magic.wdl diff --git a/tests/workflows/theiaprok/test_wf_theiaprok_illumina_se.yml b/tests/workflows/theiaprok/test_wf_theiaprok_illumina_se.yml index 88584182b..fbbd7d4fe 100644 --- a/tests/workflows/theiaprok/test_wf_theiaprok_illumina_se.yml +++ b/tests/workflows/theiaprok/test_wf_theiaprok_illumina_se.yml @@ -592,7 +592,7 @@ - path: miniwdl_run/wdl/tasks/taxon_id/contamination/task_midas.wdl md5sum: 64caaaff5910ac0036e2659434500962 - path: miniwdl_run/wdl/tasks/utilities/data_export/task_broad_terra_tools.wdl - md5sum: 4d69a6539b68503af9f3f1c2787ff920 + md5sum: 850ad97598aca5c28eb36e6a5c13c2fc - path: miniwdl_run/wdl/workflows/theiaprok/wf_theiaprok_illumina_se.wdl md5sum: fdb66b59ac886501a4ae90a25cefd633 - path: miniwdl_run/wdl/workflows/utilities/wf_merlin_magic.wdl