Merge pull request #42 from mskcc/develop

Develop

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "modules/sample/resolve-pdx"]
 	path = modules/sample/resolve-pdx
 	url = https://github.com/mskcc/resolve-pdx
+[submodule "modules/pair/helix_filters_01"]
+	path = modules/pair/helix_filters_01
+	url = https://github.com/mskcc/helix_filters_01

test/copy_outputs.yaml

@@ -0,0 +1,35 @@
+vcf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf
+bam:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam
+maf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf
+facets:
+- normal_id: foo_normal
+  tumor_id: foo_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_purity.CNCF.png
+- normal_id: bar_normal
+  tumor_id: bar_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_purity.CNCF.png
+meta:
+- class: File
+  path: /juno/work/ci/roslin-test/data/fastq/DU874145-N/SampleSheet.csv
+- class: File
+  path: /juno/work/ci/roslin-test/data/fastq/DU874145-T/SampleSheet.csv

test/copy_outputs_without_meta.yaml

@@ -0,0 +1,30 @@
+vcf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf
+bam:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam
+maf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf
+facets:
+- normal_id: foo_normal
+  tumor_id: foo_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_purity.CNCF.png
+- normal_id: bar_normal
+  tumor_id: bar_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_purity.CNCF.png

test/test.sh

@@ -1,10 +1,31 @@
 #!/bin/bash
-mkdir -p ${1}/sv/work
-mkdir -p ${1}/sv/out
-mkdir -p ${1}/non_sv/work
-mkdir -p ${1}/non_sv/out
-cwltoil --singularity --logFile ${1}/sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/sv/jobstore --tmpdir-prefix /scratch --workDir ${1}/sv/work --outdir ${1}/sv/out --maxLocalJobs 500 ../project-workflow-sv.cwl inputs.yaml &
-SV_PROCESS=$!
-cwltoil --singularity --logFile ${1}/non_sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/non_sv/jobstore --tmpdir-prefix /scratch --workDir ${1}/non_sv/work --outdir ${1}/non_sv/out --maxLocalJobs 500 ../project-workflow.cwl inputs.yaml &
-NON_SV_PROCESS=$!
-wait $SV_PROCESS $NON_SV_PROCESS
+
+set -eu
+PROCESS_LIST=()
+
+run_test() {
+	mkdir -p ${1}/${2}/work
+	mkdir -p ${1}/${2}/out
+	cwltoil --singularity --logFile ${1}/${2}/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} &
+	PROCESS_LIST+=($!)
+}
+
+# Test the SV cwl
+
+run_test $1 sv ../project-workflow-sv.cwl inputs.yaml
+
+# Test the non SV cwl
+
+run_test $1 non_sv ../project-workflow.cwl inputs.yaml
+
+# Test the copy outputs cwl
+
+run_test $1 copy_outputs ../workflows/copy_outputs.cwl copy_outputs.yaml
+
+# Test the copy outputs cwl without meta
+
+run_test $1 copy_outputs_without_meta ../workflows/copy_outputs.cwl copy_outputs_without_meta.yaml
+
+for SINGLE_PROCESS in "${PROCESS_LIST[@]}"; do
+  wait "$SINGLE_PROCESS"
+done

tools/bwa-mem/0.7.12/bwa-mem.cwl

@@ -15,8 +15,12 @@ requirements:
   DockerRequirement:
     dockerPull: mskcc/bwa_mem:0.7.12
 
+label: "run bwa mem -t 6 -M"
 doc: |
-  run bwa mem
+  bwa mem
+    args set:
+      -M
+      -t 6
 
 inputs:
 
@@ -50,6 +54,31 @@ inputs:
   output:
     type: string
 
+#
+# Non-default args used: -M -t 6
+#
+
+  M:
+    type: ['null', boolean]
+    default: true
+    doc: mark shorter split hits as secondary (for Picard/GATK compatibility)
+    inputBinding:
+      prefix: -M
+      position: 0
+
+  t:
+    type: ['null', string]
+    doc: INT number of threads [1]
+    inputBinding:
+      prefix: -t
+      position: 0
+    default: '6'
+
+#
+# Remaining arguments at default values unless
+# explicitly set
+#
+
   E:
     type: ['null', string]
     doc: INT gap extension penalty; a gap of size k cost {-O} + {-E}*k [1]
@@ -93,13 +122,6 @@ inputs:
       prefix: -B
       position: 0
 
-  M:
-    type: ['null', boolean]
-    default: true
-    doc: mark shorter split hits as secondary (for Picard/GATK compatibility)
-    inputBinding:
-      prefix: -M
-      position: 0
 
   L:
     type: ['null', string]
@@ -136,13 +158,6 @@ inputs:
       prefix: -U
       position: 0
 
-  t:
-    type: ['null', string]
-    doc: INT number of threads [1]
-    inputBinding:
-      prefix: -t
-      position: 0
-    default: '6'
 
   w:
     type: ['null', string]

workflows/copy_outputs.cwl

@@ -15,6 +15,9 @@ inputs:
   vcf: File[]
   bam: File[]
   maf: File[]
+  meta:
+    type: File[]
+    default: []
   facets:
     type:
       type: array
@@ -36,6 +39,9 @@ outputs:
   maf_dir:
     type: Directory
     outputSource: collect_maf/directory
+  meta_files:
+    type: File[]
+    outputSource: meta
   facets:
     type: Directory
     outputSource: collect_facets/directory