Merge pull request #41 from mskcc/feature/add_meta_input_for_copy_out…

…puts

Feature/add meta input for copy outputs

test/copy_outputs.yaml

@@ -0,0 +1,35 @@
+vcf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf
+bam:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam
+maf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf
+facets:
+- normal_id: foo_normal
+  tumor_id: foo_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_purity.CNCF.png
+- normal_id: bar_normal
+  tumor_id: bar_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_purity.CNCF.png
+meta:
+- class: File
+  path: /juno/work/ci/roslin-test/data/fastq/DU874145-N/SampleSheet.csv
+- class: File
+  path: /juno/work/ci/roslin-test/data/fastq/DU874145-T/SampleSheet.csv

test/copy_outputs_without_meta.yaml

@@ -0,0 +1,30 @@
+vcf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf
+bam:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam
+maf:
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf
+- class: File
+  path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf
+facets:
+- normal_id: foo_normal
+  tumor_id: foo_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads__foo_tumor.rg.md.abra.printreads_purity.CNCF.png
+- normal_id: bar_normal
+  tumor_id: bar_tumor
+  files:
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_hisens.CNCF.png
+  - class: File
+    path: /juno/work/ci/roslin-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads__bar_tumor.rg.md.abra.printreads_purity.CNCF.png

test/test.sh

@@ -1,10 +1,31 @@
 #!/bin/bash
-mkdir -p ${1}/sv/work
-mkdir -p ${1}/sv/out
-mkdir -p ${1}/non_sv/work
-mkdir -p ${1}/non_sv/out
-cwltoil --singularity --logFile ${1}/sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/sv/jobstore --tmpdir-prefix /scratch --workDir ${1}/sv/work --outdir ${1}/sv/out --maxLocalJobs 500 ../project-workflow-sv.cwl inputs.yaml &
-SV_PROCESS=$!
-cwltoil --singularity --logFile ${1}/non_sv/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/non_sv/jobstore --tmpdir-prefix /scratch --workDir ${1}/non_sv/work --outdir ${1}/non_sv/out --maxLocalJobs 500 ../project-workflow.cwl inputs.yaml &
-NON_SV_PROCESS=$!
-wait $SV_PROCESS $NON_SV_PROCESS
+
+set -eu
+PROCESS_LIST=()
+
+run_test() {
+	mkdir -p ${1}/${2}/work
+	mkdir -p ${1}/${2}/out
+	cwltoil --singularity --logFile ${1}/${2}/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} &
+	PROCESS_LIST+=($!)
+}
+
+# Test the SV cwl
+
+run_test $1 sv ../project-workflow-sv.cwl inputs.yaml
+
+# Test the non SV cwl
+
+run_test $1 non_sv ../project-workflow.cwl inputs.yaml
+
+# Test the copy outputs cwl
+
+run_test $1 copy_outputs ../workflows/copy_outputs.cwl copy_outputs.yaml
+
+# Test the copy outputs cwl without meta
+
+run_test $1 copy_outputs_without_meta ../workflows/copy_outputs.cwl copy_outputs_without_meta.yaml
+
+for SINGLE_PROCESS in "${PROCESS_LIST[@]}"; do
+  wait "$SINGLE_PROCESS"
+done

workflows/copy_outputs.cwl

@@ -15,6 +15,9 @@ inputs:
   vcf: File[]
   bam: File[]
   maf: File[]
+  meta:
+    type: File[]
+    default: []
   facets:
     type:
       type: array
@@ -36,6 +39,9 @@ outputs:
   maf_dir:
     type: Directory
     outputSource: collect_maf/directory
+  meta_files:
+    type: File[]
+    outputSource: meta
   facets:
     type: Directory
     outputSource: collect_facets/directory