From 7a80790ebea6e4d65d4ecdef1c946f268293ba9a Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Mon, 4 May 2020 07:36:43 -0400 Subject: [PATCH 01/14] Adding helix filters submodule to modules/pair dir --- .gitmodules | 3 +++ modules/pair/helix_filters_01 | 1 + 2 files changed, 4 insertions(+) create mode 160000 modules/pair/helix_filters_01 diff --git a/.gitmodules b/.gitmodules index e03b19b..abe21ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "modules/sample/resolve-pdx"] path = modules/sample/resolve-pdx url = https://github.com/mskcc/resolve-pdx +[submodule "modules/pair/helix_filters_01"] + path = modules/pair/helix_filters_01 + url = git@github.com:mskcc/helix_filters_01 diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 new file mode 160000 index 0000000..bda2c6e --- /dev/null +++ b/modules/pair/helix_filters_01 @@ -0,0 +1 @@ +Subproject commit bda2c6ef61168e3e0c170f993ca8f96da9fb48b4 From 7e3528e2d018ae70aa26df9f861b9d7070299083 Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Tue, 5 May 2020 18:15:39 -0400 Subject: [PATCH 02/14] Changed ssh call to https in .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index abe21ce..f5a955a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/mskcc/resolve-pdx [submodule "modules/pair/helix_filters_01"] path = modules/pair/helix_filters_01 - url = git@github.com:mskcc/helix_filters_01 + url = https://github.com:mskcc/helix_filters_01 From 21d35a618cf4241c13ce8e8bd12c992eb4438b42 Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Tue, 5 May 2020 18:23:05 -0400 Subject: [PATCH 03/14] Fixed .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index f5a955a..6c98352 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/mskcc/resolve-pdx [submodule "modules/pair/helix_filters_01"] path = modules/pair/helix_filters_01 - url = https://github.com:mskcc/helix_filters_01 + url = https://github.com/mskcc/helix_filters_01 From f3f58727b432f7261e93561bbd7ef9313b501a86 Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Wed, 6 May 2020 14:44:24 -0400 Subject: [PATCH 04/14] Updated submodule --- modules/pair/helix_filters_01 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 index bda2c6e..f9e6732 160000 --- a/modules/pair/helix_filters_01 +++ b/modules/pair/helix_filters_01 @@ -1 +1 @@ -Subproject commit bda2c6ef61168e3e0c170f993ca8f96da9fb48b4 +Subproject commit f9e6732490304f262d458c9fdadb31c90c28a29e From 728cb58edecd03fb6224b7c1fe7baea61427f150 Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Wed, 6 May 2020 14:54:39 -0400 Subject: [PATCH 05/14] Updated submodule --- modules/pair/helix_filters_01 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 index f9e6732..ffac2e2 160000 --- a/modules/pair/helix_filters_01 +++ b/modules/pair/helix_filters_01 @@ -1 +1 @@ -Subproject commit f9e6732490304f262d458c9fdadb31c90c28a29e +Subproject commit ffac2e27b64751685ddeb02ecb89b81485f08e63 From 841b1c4888d50e564a7d353440ec6e065b327e86 Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Wed, 6 May 2020 15:27:42 -0400 Subject: [PATCH 06/14] Updated submodule --- modules/pair/helix_filters_01 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 index ffac2e2..62efb19 160000 --- a/modules/pair/helix_filters_01 +++ b/modules/pair/helix_filters_01 @@ -1 +1 @@ -Subproject commit ffac2e27b64751685ddeb02ecb89b81485f08e63 +Subproject commit 62efb193521f76680499825c85ff95dd8bf1eecd From 052cb78faaa5655160b1872af5e5d44bd744a575 Mon Sep 17 00:00:00 2001 From: "Nicholas D. Socci" Date: Thu, 7 May 2020 19:09:01 -0400 Subject: [PATCH 07/14] Exegesis: explicitly list BWA arguments used Add some comments so mortal humans can see what arguments are being used when `bwa mem` is being run. --- tools/bwa-mem/0.7.12/bwa-mem.cwl | 45 +++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/tools/bwa-mem/0.7.12/bwa-mem.cwl b/tools/bwa-mem/0.7.12/bwa-mem.cwl index bca6473..423454b 100644 --- a/tools/bwa-mem/0.7.12/bwa-mem.cwl +++ b/tools/bwa-mem/0.7.12/bwa-mem.cwl @@ -15,8 +15,12 @@ requirements: DockerRequirement: dockerPull: mskcc/bwa_mem:0.7.12 +label: "run bwa mem -t 6 -M" doc: | - run bwa mem + bwa mem + args set: + -M + -t 6 inputs: @@ -50,6 +54,31 @@ inputs: output: type: string +# +# Non-default args used: -M -t 6 +# + + M: + type: ['null', boolean] + default: true + doc: mark shorter split hits as secondary (for Picard/GATK compatibility) + inputBinding: + prefix: -M + position: 0 + + t: + type: ['null', string] + doc: INT number of threads [1] + inputBinding: + prefix: -t + position: 0 + default: '6' + +# +# Remaining arguments at default values unless +# explicitly set +# + E: type: ['null', string] doc: INT gap extension penalty; a gap of size k cost {-O} + {-E}*k [1] @@ -93,13 +122,6 @@ inputs: prefix: -B position: 0 - M: - type: ['null', boolean] - default: true - doc: mark shorter split hits as secondary (for Picard/GATK compatibility) - inputBinding: - prefix: -M - position: 0 L: type: ['null', string] @@ -136,13 +158,6 @@ inputs: prefix: -U position: 0 - t: - type: ['null', string] - doc: INT number of threads [1] - inputBinding: - prefix: -t - position: 0 - default: '6' w: type: ['null', string] From a668e384e89ea9be2be245af563b64b626bf7457 Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Mon, 11 May 2020 15:39:56 -0400 Subject: [PATCH 08/14] Updated helix filters --- modules/pair/helix_filters_01 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 index 62efb19..e70f3f5 160000 --- a/modules/pair/helix_filters_01 +++ b/modules/pair/helix_filters_01 @@ -1 +1 @@ -Subproject commit 62efb193521f76680499825c85ff95dd8bf1eecd +Subproject commit e70f3f56cf47a1c15161e9ff2b2f34cfdb85429d From 3a08e58a903e291f95e7b2e2838af7725dd3ce9d Mon Sep 17 00:00:00 2001 From: Allan Bolipata Date: Mon, 11 May 2020 17:18:05 -0400 Subject: [PATCH 09/14] Updated helix filters submodule to most recent master branch --- modules/pair/helix_filters_01 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 index e70f3f5..5488b2a 160000 --- a/modules/pair/helix_filters_01 +++ b/modules/pair/helix_filters_01 @@ -1 +1 @@ -Subproject commit e70f3f56cf47a1c15161e9ff2b2f34cfdb85429d +Subproject commit 5488b2ac221311ef012f82d23a271d8a8034f175 From 36a8fb575491e7f1323f92758799f1ea26d8b0ca Mon Sep 17 00:00:00 2001 From: Nikhil Date: Mon, 11 May 2020 18:45:37 -0400 Subject: [PATCH 10/14] Added optional meta input into copy_outputs --- workflows/copy_outputs.cwl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workflows/copy_outputs.cwl b/workflows/copy_outputs.cwl index e2e3854..ce4eb13 100644 --- a/workflows/copy_outputs.cwl +++ b/workflows/copy_outputs.cwl @@ -15,6 +15,9 @@ inputs: vcf: File[] bam: File[] maf: File[] + meta: + type: File[] + default: [] facets: type: type: array @@ -36,6 +39,9 @@ outputs: maf_dir: type: Directory outputSource: collect_maf/directory + meta_files: + type: File[] + outputSource: meta facets: type: Directory outputSource: collect_facets/directory From 2ab668b0a24a1ec34d0a1768e45caf1f5715a403 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Mon, 11 May 2020 18:46:17 -0400 Subject: [PATCH 11/14] Added extra input files for copy_outputs tests --- test/copy_outputs.yaml | 35 +++++++++++++++++++++++++++++ test/copy_outputs_without_meta.yaml | 30 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 test/copy_outputs.yaml create mode 100644 test/copy_outputs_without_meta.yaml diff --git a/test/copy_outputs.yaml b/test/copy_outputs.yaml new file mode 100644 index 0000000..eea5e7f --- /dev/null +++ b/test/copy_outputs.yaml @@ -0,0 +1,35 @@ +vcf: +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf +bam: +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam +maf: +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf +facets: +- normal_id: foo_normal + tumor_id: foo_tumor + files: + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_hisens.CNCF.png + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_purity.CNCF.png +- normal_id: bar_normal + tumor_id: bar_tumor + files: + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_hisens.CNCF.png + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_purity.CNCF.png +meta: +- class: File + path: /juno/work/ci/roslin-test/data/fastq/DU874145-N/SampleSheet.csv +- class: File + path: /juno/work/ci/roslin-test/data/fastq/DU874145-T/SampleSheet.csv \ No newline at end of file diff --git a/test/copy_outputs_without_meta.yaml b/test/copy_outputs_without_meta.yaml new file mode 100644 index 0000000..92ecf4f --- /dev/null +++ b/test/copy_outputs_without_meta.yaml @@ -0,0 +1,30 @@ +vcf: +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf +bam: +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam +maf: +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf +- class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf +facets: +- normal_id: foo_normal + tumor_id: foo_tumor + files: + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_hisens.CNCF.png + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_purity.CNCF.png +- normal_id: bar_normal + tumor_id: bar_tumor + files: + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_hisens.CNCF.png + - class: File + path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_purity.CNCF.png \ No newline at end of file From d9cf174e690bd9538db7706ff5e5ee462d266a02 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Mon, 11 May 2020 18:46:57 -0400 Subject: [PATCH 12/14] Refactored test script and added more tests --- test/test.sh | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/test/test.sh b/test/test.sh index 28b0970..25c22d6 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,10 +1,29 @@ #!/bin/bash -mkdir -p ${1}/sv/work -mkdir -p ${1}/sv/out -mkdir -p ${1}/non_sv/work -mkdir -p ${1}/non_sv/out -cwltoil --singularity --logFile ${1}/sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/sv/jobstore --tmpdir-prefix /scratch --workDir ${1}/sv/work --outdir ${1}/sv/out --maxLocalJobs 500 ../project-workflow-sv.cwl inputs.yaml & -SV_PROCESS=$! -cwltoil --singularity --logFile ${1}/non_sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/non_sv/jobstore --tmpdir-prefix /scratch --workDir ${1}/non_sv/work --outdir ${1}/non_sv/out --maxLocalJobs 500 ../project-workflow.cwl inputs.yaml & -NON_SV_PROCESS=$! -wait $SV_PROCESS $NON_SV_PROCESS +run_test() { + mkdir -p ${1}/${2}/work + mkdir -p ${1}/${2}/out + cwltoil --singularity --logFile ${1}/sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & + return $! +} + +# Test the SV cwl + +run_test $1 sv ../project-workflow-sv inputs.yaml +SV_PROCESS=$? + +# Test the non SV cwl + +run_test $1 non_sv ../project-workflow.cwl inputs.yaml +NON_SV_PROCESS=$? + +# Test the copy outputs cwl + +run_test $1 copy_outputs ../workflows/copy_outputs.cwl copy_outputs.yaml +COPY_OUTPUTS_PROCESS=$? + +# Test the copy outputs cwl without meta + +run_test $1 copy_outputs_without_meta ../workflows/copy_outputs.cwl copy_outputs_without_meta.yaml +COPY_OUTPUTS_WITHOUT_META_PROCESS=$? + +wait $SV_PROCESS $NON_SV_PROCESS $COPY_OUTPUTS_PROCESS $COPY_OUTPUTS_WITHOUT_META_PROCESS \ No newline at end of file From 161f772e1bb230abdaa70cc19c528e9d81f54a68 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Mon, 11 May 2020 20:45:48 -0400 Subject: [PATCH 13/14] A few fixes for the test script - Fixed typo in toil_log output - Fixed script to exit properly when running jobs in parallel --- test/test.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/test.sh b/test/test.sh index 25c22d6..3426f9b 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,29 +1,31 @@ #!/bin/bash + +set -eu +PROCESS_LIST=() + run_test() { mkdir -p ${1}/${2}/work mkdir -p ${1}/${2}/out - cwltoil --singularity --logFile ${1}/sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & - return $! + cwltoil --singularity --logFile ${1}/${2}/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & + PROCESS_LIST+=($!) } # Test the SV cwl run_test $1 sv ../project-workflow-sv inputs.yaml -SV_PROCESS=$? # Test the non SV cwl run_test $1 non_sv ../project-workflow.cwl inputs.yaml -NON_SV_PROCESS=$? # Test the copy outputs cwl run_test $1 copy_outputs ../workflows/copy_outputs.cwl copy_outputs.yaml -COPY_OUTPUTS_PROCESS=$? # Test the copy outputs cwl without meta run_test $1 copy_outputs_without_meta ../workflows/copy_outputs.cwl copy_outputs_without_meta.yaml -COPY_OUTPUTS_WITHOUT_META_PROCESS=$? -wait $SV_PROCESS $NON_SV_PROCESS $COPY_OUTPUTS_PROCESS $COPY_OUTPUTS_WITHOUT_META_PROCESS \ No newline at end of file +for SINGLE_PROCESS in "${PROCESS_LIST[@]}"; do + wait "$SINGLE_PROCESS" +done From f9f0ed6c691db425759b8fd1d809bae1eb9524f5 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 12 May 2020 12:05:39 -0400 Subject: [PATCH 14/14] Slight typo fix --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 3426f9b..365ec57 100755 --- a/test/test.sh +++ b/test/test.sh @@ -12,7 +12,7 @@ run_test() { # Test the SV cwl -run_test $1 sv ../project-workflow-sv inputs.yaml +run_test $1 sv ../project-workflow-sv.cwl inputs.yaml # Test the non SV cwl