From 6c271dd1ae0fcd511b43a3c83506991567f4d83b Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Thu, 3 Jun 2021 15:42:59 -0400 Subject: [PATCH 01/12] Add secondary files for ref-fasta --- tools/cmo-utils/1.9.15/cmo-fillout.cwl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/cmo-utils/1.9.15/cmo-fillout.cwl b/tools/cmo-utils/1.9.15/cmo-fillout.cwl index 6ff50da..ffc521b 100644 --- a/tools/cmo-utils/1.9.15/cmo-fillout.cwl +++ b/tools/cmo-utils/1.9.15/cmo-fillout.cwl @@ -42,6 +42,14 @@ inputs: doc: Reference assembly file of BAM files, e.g. hg19/grch37/b37 inputBinding: prefix: --ref-fasta + secondaryFiles: + - .amb + - .ann + - .bwt + - .pac + - .sa + - .fai + - ^.dict output: type: ['null', string] From 7437b300f6f91e0d2674a1dc6bb81cce79e4e63b Mon Sep 17 00:00:00 2001 From: Nikhil Date: Thu, 17 Jun 2021 13:46:06 -0400 Subject: [PATCH 02/12] Split pairs into tumor and normal --- modules/pair/alignment-pair.cwl | 67 +++++++++++++++-------- modules/pair/maf-processing-pair.cwl | 71 +++++++++++++++--------- modules/pair/realignment.cwl | 28 ++-------- tools/mutect/1.1.4/mutect.cwl | 1 - workflows/pair-workflow-sv.cwl | 82 ++++++++++++++++++---------- workflows/pair-workflow.cwl | 76 +++++++++++++++++--------- 6 files changed, 197 insertions(+), 128 deletions(-) diff --git a/modules/pair/alignment-pair.cwl b/modules/pair/alignment-pair.cwl index 46fd528..2c4257d 100644 --- a/modules/pair/alignment-pair.cwl +++ b/modules/pair/alignment-pair.cwl @@ -10,26 +10,6 @@ requirements: inputs: - pair: - type: - type: array - items: - type: record - fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string genome: string intervals: string[] opt_dup_pix_dist: string @@ -76,6 +56,42 @@ inputs: - .fai - ^.dict conpair_markers_bed: string + tumor: + type: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + normal: + type: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string outputs: @@ -148,7 +164,9 @@ steps: sample_alignment: run: ../../workflows/sample-workflow.cwl in: - sample: pair + sample: + source: [tumor, normal] + linkMerge: merge_flattened genome: genome opt_dup_pix_dist: opt_dup_pix_dist gatk_jar_path: gatk_jar_path @@ -164,7 +182,6 @@ steps: realignment: run: realignment.cwl in: - pair: pair bams: sample_alignment/bam hapmap: hapmap dbsnp: dbsnp @@ -175,4 +192,10 @@ steps: ref_fasta: ref_fasta intervals: intervals abra_ram_min: abra_ram_min + tumor: tumor + normal: normal + normal_name: + valueFrom: ${ return inputs.normal.ID; } + tumor_name: + valueFrom: ${ return inputs.tumor.ID; } out: [outbams, covint_list, covint_bed, qual_metrics, qual_pdf] diff --git a/modules/pair/maf-processing-pair.cwl b/modules/pair/maf-processing-pair.cwl index e764c6f..16d4c68 100644 --- a/modules/pair/maf-processing-pair.cwl +++ b/modules/pair/maf-processing-pair.cwl @@ -11,26 +11,43 @@ requirements: inputs: - pair: + tumor: type: - type: array - items: type: record fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + normal: + type: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + bams: type: File[] secondaryFiles: @@ -72,9 +89,10 @@ outputs: steps: create_pairing_file: in: - pair: pair + tumor: tumor + normal: normal echoString: - valueFrom: ${ return inputs.pair[1].ID + "\t" + inputs.pair[0].ID; } + valueFrom: ${ return inputs.normal.ID + "\t" + inputs.tumor.ID; } output_filename: valueFrom: ${ return "tn_pairing_file.txt"; } out: [ pairfile ] @@ -89,13 +107,16 @@ steps: DockerRequirement: dockerPull: alpine:3.8 inputs: - pair: + tumor: + type: + type: record + fields: + ID: string + normal: type: - type: array - items: - type: record - fields: - ID: string + type: record + fields: + ID: string echoString: type: string inputBinding: diff --git a/modules/pair/realignment.cwl b/modules/pair/realignment.cwl index d1134da..b38d8b7 100644 --- a/modules/pair/realignment.cwl +++ b/modules/pair/realignment.cwl @@ -13,26 +13,8 @@ inputs: type: File[] secondaryFiles: - ^.bai - pair: - type: - type: array - items: - type: record - fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string + normal_name: string + tumor_name: string ref_fasta: type: File secondaryFiles: @@ -116,7 +98,6 @@ steps: gatk_find_covered_intervals: run: ../../tools/gatk.FindCoveredIntervals/3.3-0/gatk.FindCoveredIntervals.cwl in: - pair: pair intervals_list: intervals reference_sequence: ref_fasta coverage_threshold: @@ -132,9 +113,10 @@ steps: combine_intervals: in: files: gatk_find_covered_intervals/fci_list - pair: pair + normal_name: normal_name + tumor_name: tumor_name output_filename: - valueFrom: ${ return inputs.pair[0].ID + "." + inputs.pair[1].ID + ".fci.list"; } + valueFrom: ${ return inputs.tumor_name + "." + inputs.normal_name + ".fci.list"; } out: [mergedfile] run: class: CommandLineTool diff --git a/tools/mutect/1.1.4/mutect.cwl b/tools/mutect/1.1.4/mutect.cwl index 5fd3440..d2c9496 100644 --- a/tools/mutect/1.1.4/mutect.cwl +++ b/tools/mutect/1.1.4/mutect.cwl @@ -36,7 +36,6 @@ doc: | None inputs: - analysis_type: type: - 'null' diff --git a/workflows/pair-workflow-sv.cwl b/workflows/pair-workflow-sv.cwl index 5ce4db0..e49efb4 100644 --- a/workflows/pair-workflow-sv.cwl +++ b/workflows/pair-workflow-sv.cwl @@ -103,26 +103,42 @@ inputs: opt_dup_pix_dist: string abra_ram_min: int gatk_jar_path: string - pair: + tumor: type: - type: array - items: - type: record - fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + normal: + type: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string outputs: @@ -273,7 +289,8 @@ steps: in: runparams: runparams db_files: db_files - pair: pair + tumor: tumor + normal: normal genome: valueFrom: ${ return inputs.runparams.genome } intervals: @@ -310,16 +327,17 @@ steps: runparams: runparams db_files: db_files bams: alignment/bams - pair: pair + tumor: tumor + normal: normal normal_bam: valueFrom: ${ return inputs.bams[1]; } tumor_bam: valueFrom: ${ return inputs.bams[0]; } bed: alignment/bed normal_sample_name: - valueFrom: ${ return inputs.pair[1].ID; } + valueFrom: ${ return inputs.normal.ID; } tumor_sample_name: - valueFrom: ${ return inputs.pair[0].ID; } + valueFrom: ${ return inputs.tumor.ID; } dbsnp: dbsnp cosmic: cosmic mutect_dcov: @@ -346,7 +364,8 @@ steps: db_files: db_files exac_filter: exac_filter bams: alignment/bams - pair: pair + tumor: tumor + normal: normal normal_bam: valueFrom: ${ return inputs.bams[1]; } tumor_bam: @@ -354,9 +373,9 @@ steps: genome: valueFrom: ${ return inputs.runparams.genome } normal_sample_name: - valueFrom: ${ return inputs.pair[1].ID; } + valueFrom: ${ return inputs.normal.ID; } tumor_sample_name: - valueFrom: ${ return inputs.pair[0].ID; } + valueFrom: ${ return inputs.tumor.ID; } ref_fasta: ref_fasta vep_path: valueFrom: ${ return inputs.db_files.vep_path } @@ -376,7 +395,8 @@ steps: db_files: db_files bams: alignment/bams annotate_vcf: variant_calling/annotate_vcf - pair: pair + tumor: tumor + normal: normal genome: valueFrom: ${ return inputs.runparams.genome } ref_fasta: ref_fasta @@ -388,9 +408,9 @@ steps: vep_data: valueFrom: ${ return inputs.db_files.vep_data } normal_sample_name: - valueFrom: ${ return inputs.pair[1].ID; } + valueFrom: ${ return inputs.normal.ID; } tumor_sample_name: - valueFrom: ${ return inputs.pair[0].ID; } + valueFrom: ${ return inputs.tumor.ID; } curated_bams: curated_bams hotspot_list: valueFrom: ${ return inputs.db_files.hotspot_list } @@ -399,6 +419,8 @@ steps: run: ../tools/format-output/pair-output.cwl in: runparams: runparams - pair: pair + pair: + source: [tumor, normal] + linkMerge: merge_flattened bams: alignment/bams out: [ genome, assay, pi, pi_email, project_prefix, normal_sample_name, tumor_sample_name, normal_bam, tumor_bam ] diff --git a/workflows/pair-workflow.cwl b/workflows/pair-workflow.cwl index c744808..29661bd 100644 --- a/workflows/pair-workflow.cwl +++ b/workflows/pair-workflow.cwl @@ -102,26 +102,43 @@ inputs: opt_dup_pix_dist: string abra_ram_min: int gatk_jar_path: string - pair: + tumor: type: - type: array - items: - type: record - fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + normal: + type: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + outputs: @@ -257,7 +274,8 @@ steps: in: runparams: runparams db_files: db_files - pair: pair + tumor: tumor + normal: normal genome: valueFrom: ${ return inputs.runparams.genome } intervals: @@ -293,16 +311,17 @@ steps: runparams: runparams db_files: db_files bams: alignment/bams - pair: pair + tumor: tumor + normal: normal normal_bam: valueFrom: ${ return inputs.bams[1]; } tumor_bam: valueFrom: ${ return inputs.bams[0]; } bed: alignment/bed normal_sample_name: - valueFrom: ${ return inputs.pair[1].ID; } + valueFrom: ${ return inputs.normal.ID; } tumor_sample_name: - valueFrom: ${ return inputs.pair[0].ID; } + valueFrom: ${ return inputs.tumor.ID; } dbsnp: dbsnp cosmic: cosmic mutect_dcov: @@ -327,8 +346,9 @@ steps: runparams: runparams db_files: db_files bams: alignment/bams + tumor: tumor + normal: normal annotate_vcf: variant_calling/annotate_vcf - pair: pair genome: valueFrom: ${ return inputs.runparams.genome } ref_fasta: ref_fasta @@ -340,9 +360,9 @@ steps: vep_data: valueFrom: ${ return inputs.db_files.vep_data } normal_sample_name: - valueFrom: ${ return inputs.pair[1].ID; } + valueFrom: ${ return inputs.normal.ID; } tumor_sample_name: - valueFrom: ${ return inputs.pair[0].ID; } + valueFrom: ${ return inputs.tumor.ID; } curated_bams: curated_bams hotspot_list: valueFrom: ${ return inputs.db_files.hotspot_list } @@ -351,7 +371,9 @@ steps: run: ../tools/format-output/pair-output.cwl in: runparams: runparams - pair: pair + pair: + source: [tumor, normal] + linkMerge: merge_flattened bams: alignment/bams out: [ genome, assay, pi, pi_email, project_prefix, normal_sample_name, tumor_sample_name, normal_bam, tumor_bam ] From 7213b52c25d0c61cbbbcdfbf457c35972aa34ca7 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Thu, 17 Jun 2021 13:46:36 -0400 Subject: [PATCH 03/12] Fixed pair handling in project level cwls --- project-workflow-sv.cwl | 61 +++++++++++++++++++++++++++-------------- project-workflow.cwl | 61 +++++++++++++++++++++++++++-------------- 2 files changed, 80 insertions(+), 42 deletions(-) diff --git a/project-workflow-sv.cwl b/project-workflow-sv.cwl index 54b81a4..08fd643 100644 --- a/project-workflow-sv.cwl +++ b/project-workflow-sv.cwl @@ -105,28 +105,46 @@ inputs: abra_ram_min: int scripts_bin: string gatk_jar_path: string - pairs: + tumors: type: type: array items: - type: array - items: - type: record - fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + normals: + type: + type: array + items: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string outputs: @@ -257,11 +275,12 @@ steps: exac_filter: exac_filter curated_bams: curated_bams cosmic: cosmic - pair: pairs + tumor: tumors + normal: normals ref_fasta: ref_fasta mouse_fasta: mouse_fasta out: [normal_bam,tumor_bam,clstats1,clstats2,md_metrics,as_metrics,hs_metrics,insert_metrics,insert_pdf,per_target_coverage,qual_metrics,qual_pdf,doc_basecounts,gcbias_pdf,gcbias_metrics,gcbias_summary,conpair_pileups,mutect_vcf,mutect_callstats,vardict_vcf,combine_vcf,annotate_vcf,vardict_norm_vcf,mutect_norm_vcf,snp_pileup,merged_file_unfiltered,merged_file,maf_file,portal_file,maf,genome,assay,pi,pi_email,project_prefix,normal_sample_name,tumor_sample_name] - scatter: [pair] + scatter: [tumor, normal] scatterMethod: dotproduct generate_qc: diff --git a/project-workflow.cwl b/project-workflow.cwl index b788d0f..1b2ece9 100644 --- a/project-workflow.cwl +++ b/project-workflow.cwl @@ -104,28 +104,46 @@ inputs: abra_ram_min: int scripts_bin: string gatk_jar_path: string - pairs: + tumors: type: type: array items: - type: array - items: - type: record - fields: - CN: string - LB: string - ID: string - PL: string - PU: string[] - R1: File[] - R2: File[] - zR1: File[] - zR2: File[] - bam: File[] - RG_ID: string[] - adapter: string - adapter2: string - bwa_output: string + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string + normals: + type: + type: array + items: + type: record + fields: + CN: string + LB: string + ID: string + PL: string + PU: string[] + R1: File[] + R2: File[] + zR1: File[] + zR2: File[] + bam: File[] + RG_ID: string[] + adapter: string + adapter2: string + bwa_output: string outputs: @@ -243,11 +261,12 @@ steps: exac_filter: exac_filter curated_bams: curated_bams cosmic: cosmic - pair: pairs + tumor: tumors + normal: normals ref_fasta: ref_fasta mouse_fasta: mouse_fasta out: [normal_bam,tumor_bam,clstats1,clstats2,md_metrics,as_metrics,hs_metrics,insert_metrics,insert_pdf,per_target_coverage,qual_metrics,qual_pdf,doc_basecounts,gcbias_pdf,gcbias_metrics,gcbias_summary,conpair_pileups,mutect_vcf,mutect_callstats,vardict_vcf,combine_vcf,annotate_vcf,vardict_norm_vcf,mutect_norm_vcf,snp_pileup,maf,genome,assay,pi,pi_email,project_prefix,normal_sample_name,tumor_sample_name] - scatter: [pair] + scatter: [tumor, normal] scatterMethod: dotproduct generate_qc: From 0378287ad10f8d8e0a0945c74c7737edb0a9d8a2 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Thu, 17 Jun 2021 13:47:32 -0400 Subject: [PATCH 04/12] Fixed test inputs.yaml --- test/inputs.yaml | 167 ++++++++++++++++++++++++----------------------- 1 file changed, 84 insertions(+), 83 deletions(-) diff --git a/test/inputs.yaml b/test/inputs.yaml index 09f3580..1e4d8c9 100644 --- a/test/inputs.yaml +++ b/test/inputs.yaml @@ -129,89 +129,90 @@ indels_1000g: mouse_fasta: class: File path: /juno/work/ci/resources/genomes/GRCm38/GRCm38.fasta -pairs: -- - CN: MSKCC - ID: foo_tumor - LB: foo_tumor_1 - PL: Illumina - PU: - - foo_tumor_1-foo_tumor - R1: [] - R2: [] - RG_ID: - - foo_tumor_1-foo_tumorPE - adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG - adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT - bam: [] - bwa_output: foo_tumor.bam - group: Group_2 - zR1: - - class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R1_001.fastq.gz - zR2: - - class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R2_001.fastq.gz - - CN: MSKCC - ID: foo_normal - LB: foo_normal_1 - PL: Illumina - PU: - - foo_normal_1-foo_normal - R1: [] - R2: [] - RG_ID: - - foo_normal_1-foo_normalPE - adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG - adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT - bam: - - class: File - path: /juno/work/ci/argos-test/data/SampleWorkflowPair0Sample1/SampleWorkflow/foo_normal.rg.md.bam - bwa_output: foo_normal.bam - group: Group_2 - zR1: [] - zR2: [] -- - CN: MSKCC - ID: bar_tumor - LB: bar_tumor_1 - PL: Illumina - PU: - - bar_tumor_1-bar_tumor - R1: - - class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R1_001.fastq.gz - R2: - - class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R2_001.fastq.gz - RG_ID: - - bar_tumor_1-bar_tumorPE - adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG - adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT - bam: [] - bwa_output: bar_tumor.bam - group: Group_2 - zR1: [] - zR2: [] - - CN: MSKCC - ID: bar_normal - LB: bar_normal_1 - PL: Illumina - PU: - - bar_normal_1-bar_normal - R1: - - class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-N/DU874145-N_IGO_00000_TEST_L001_R1_001.fastq.gz - R2: - - class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-N/DU874145-N_IGO_00000_TEST_L001_R2_001.fastq.gz - RG_ID: - - bar_normal_1-bar_normalPE - adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG - adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT - bam: [] - bwa_output: bar_normal.bam - group: Group_2 - zR1: [] - zR2: [] +tumors: +- CN: MSKCC + ID: foo_tumor + LB: foo_tumor_1 + PL: Illumina + PU: + - foo_tumor_1-foo_tumor + R1: [] + R2: [] + RG_ID: + - foo_tumor_1-foo_tumorPE + adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG + adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT + bam: [] + bwa_output: foo_tumor.bam + group: Group_2 + zR1: + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R1_001.fastq.gz + zR2: + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R2_001.fastq.gz +- CN: MSKCC + ID: bar_tumor + LB: bar_tumor_1 + PL: Illumina + PU: + - bar_tumor_1-bar_tumor + R1: + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R1_001.fastq.gz + R2: + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-T/DU874145-T_IGO_00000_TEST_L001_R2_001.fastq.gz + RG_ID: + - bar_tumor_1-bar_tumorPE + adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG + adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT + bam: [] + bwa_output: bar_tumor.bam + group: Group_2 + zR1: [] + zR2: [] +normals: +- CN: MSKCC + ID: foo_normal + LB: foo_normal_1 + PL: Illumina + PU: + - foo_normal_1-foo_normal + R1: [] + R2: [] + RG_ID: + - foo_normal_1-foo_normalPE + adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG + adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT + bam: + - class: File + path: /juno/work/ci/argos-test/data/SampleWorkflowPair0Sample1/SampleWorkflow/foo_normal.rg.md.bam + bwa_output: foo_normal.bam + group: Group_2 + zR1: [] + zR2: [] +- CN: MSKCC + ID: bar_normal + LB: bar_normal_1 + PL: Illumina + PU: + - bar_normal_1-bar_normal + R1: + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-N/DU874145-N_IGO_00000_TEST_L001_R1_001.fastq.gz + R2: + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-N/DU874145-N_IGO_00000_TEST_L001_R2_001.fastq.gz + RG_ID: + - bar_normal_1-bar_normalPE + adapter: AGATCGGAAGAGCACACGTCTGAACTCCAGTCACATGAGCATCTCGTATGCCGTCTTCTGCTTG + adapter2: AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGTAGATCTCGGTGGTCGCCGTATCATT + bam: [] + bwa_output: bar_normal.bam + group: Group_2 + zR1: [] + zR2: [] ref_fasta: class: File path: /juno/work/ci/resources/genomes/GRCh37/fasta/b37.fasta From 971ad2c57c6aa6ca400eeedc78792323988c21d9 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Fri, 1 Apr 2022 16:36:00 -0400 Subject: [PATCH 05/12] Removed old helix_filters_01 module --- .gitmodules | 3 --- modules/pair/helix_filters_01 | 1 - 2 files changed, 4 deletions(-) delete mode 160000 modules/pair/helix_filters_01 diff --git a/.gitmodules b/.gitmodules index 75aaafc..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "modules/pair/helix_filters_01"] - path = modules/pair/helix_filters_01 - url = https://github.com/mskcc/helix_filters_01 diff --git a/modules/pair/helix_filters_01 b/modules/pair/helix_filters_01 deleted file mode 160000 index 647953d..0000000 --- a/modules/pair/helix_filters_01 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 647953d576fbf60161474a479cf0e88be30c3855 From c6355770ec67ee17b5a5cf07a2b466a4ced60595 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Mon, 4 Apr 2022 15:50:47 -0400 Subject: [PATCH 06/12] Updated test TOIL command --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 365ec57..3fe8cb2 100755 --- a/test/test.sh +++ b/test/test.sh @@ -6,7 +6,7 @@ PROCESS_LIST=() run_test() { mkdir -p ${1}/${2}/work mkdir -p ${1}/${2}/out - cwltoil --singularity --logFile ${1}/${2}/toil_log.log --batchSystem lsf --disable-user-provenance --disable-host-provenance --debug --clean never --disableCaching --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & + toil-cwl-runner --singularity --logFile ${1}/${2}/toil_log.log --coalesceStatusCalls --debug --clean never --disableCaching --batchSystem lsf --doubleMem --disable-user-provenance --disable-host-provenance --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD SINGULARITY_DOCKER_USERNAME SINGULARITY_DOCKER_PASSWORD SINGULARITYENV_LC_ALL --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & PROCESS_LIST+=($!) } From ef297689d0c80530ce2dd3beda8293ea68b5b7a1 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Wed, 6 Apr 2022 14:27:40 -0400 Subject: [PATCH 07/12] Updated tmp directory path --- test/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 3fe8cb2..cb76bdc 100755 --- a/test/test.sh +++ b/test/test.sh @@ -6,7 +6,8 @@ PROCESS_LIST=() run_test() { mkdir -p ${1}/${2}/work mkdir -p ${1}/${2}/out - toil-cwl-runner --singularity --logFile ${1}/${2}/toil_log.log --coalesceStatusCalls --debug --clean never --disableCaching --batchSystem lsf --doubleMem --disable-user-provenance --disable-host-provenance --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD SINGULARITY_DOCKER_USERNAME SINGULARITY_DOCKER_PASSWORD SINGULARITYENV_LC_ALL --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix /scratch --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & + mkdir -p ${1}/${2}/tmp + toil-cwl-runner --singularity --logFile ${1}/${2}/toil_log.log --coalesceStatusCalls --debug --clean never --disableCaching --batchSystem lsf --doubleMem --disable-user-provenance --disable-host-provenance --preserve-environment PATH TMPDIR TOIL_LSF_ARGS SINGULARITY_PULLDIR SINGULARITY_CACHEDIR PWD SINGULARITY_DOCKER_USERNAME SINGULARITY_DOCKER_PASSWORD SINGULARITYENV_LC_ALL --defaultMemory 8G --maxCores 16 --maxDisk 128G --maxMemory 256G --not-strict --realTimeLogging --jobStore ${1}/${2}/jobstore --tmpdir-prefix ${1}/${2}/tmp --workDir ${1}/${2}/work --outdir ${1}/${2}/out --maxLocalJobs 500 ${3} ${4} & PROCESS_LIST+=($!) } From 92b9cef98cb9915036817cca3f284fcfe9637fa0 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Fri, 8 Apr 2022 17:00:05 -0400 Subject: [PATCH 08/12] Fixed duplicate keys --- project-workflow.cwl | 1 - 1 file changed, 1 deletion(-) diff --git a/project-workflow.cwl b/project-workflow.cwl index 1b2ece9..bbf71b0 100644 --- a/project-workflow.cwl +++ b/project-workflow.cwl @@ -283,7 +283,6 @@ steps: tumor_bams: pair_process/tumor_bam normal_sample_names: pair_process/normal_sample_name tumor_sample_names: pair_process/tumor_sample_name - project_prefix_list: pair_process/project_prefix genome_list: pair_process/genome assay_list: pair_process/assay pi_list: pair_process/pi From b09f67338d85a0df4708b338d1da00c1d42fb7f0 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Sun, 10 Apr 2022 17:39:00 -0400 Subject: [PATCH 09/12] Update test.sh --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index cb76bdc..8492b7f 100755 --- a/test/test.sh +++ b/test/test.sh @@ -17,7 +17,7 @@ run_test $1 sv ../project-workflow-sv.cwl inputs.yaml # Test the non SV cwl -run_test $1 non_sv ../project-workflow.cwl inputs.yaml +# run_test $1 non_sv ../project-workflow.cwl inputs.yaml # Test the copy outputs cwl From 5446407607a8463134f3855a20d702a4531a625c Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 28 Jun 2022 09:34:48 -0400 Subject: [PATCH 10/12] Added pileup directory --- workflows/copy_outputs.cwl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/workflows/copy_outputs.cwl b/workflows/copy_outputs.cwl index 5875219..bc92aa2 100644 --- a/workflows/copy_outputs.cwl +++ b/workflows/copy_outputs.cwl @@ -15,6 +15,7 @@ inputs: vcf: File[] bam: File[] maf: File[] + pileup: File[] meta: type: File[] default: [] @@ -30,6 +31,9 @@ outputs: maf_dir: type: Directory outputSource: collect_maf/directory + pileup_dir: + type: Directory + outputSource: collect_pileup/directory meta_files: type: File[] outputSource: meta @@ -57,4 +61,11 @@ steps: output_directory_name: valueFrom: ${ return "maf"; } out: [directory] + collect_pileup: + run: ../tools/consolidate-files/consolidate-files.cwl + in: + files: pileup + output_directory_name: + valueFrom: ${ return "pileup"; } + out: [directory] From bf01197693ed43c57b95ab135ee90fc5b80a4690 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 28 Jun 2022 09:35:10 -0400 Subject: [PATCH 11/12] Added test for new pileup input --- test/copy_outputs.yaml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/copy_outputs.yaml b/test/copy_outputs.yaml index 9e8a712..1d28f72 100644 --- a/test/copy_outputs.yaml +++ b/test/copy_outputs.yaml @@ -1,20 +1,25 @@ vcf: -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf bam: -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam maf: -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf meta: -- class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-N/SampleSheet.csv -- class: File - path: /juno/work/ci/argos-test/data/fastq/DU874145-T/SampleSheet.csv \ No newline at end of file + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-N/SampleSheet.csv + - class: File + path: /juno/work/ci/argos-test/data/fastq/DU874145-T/SampleSheet.csv +pileup: + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.rg.md.pileup + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.pileup From 23e37457a70dc3475f2243c258b90a79d99ce535 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 19 Jul 2022 19:57:54 -0400 Subject: [PATCH 12/12] Added pileup to test yaml --- test/copy_outputs_without_meta.yaml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/test/copy_outputs_without_meta.yaml b/test/copy_outputs_without_meta.yaml index 83a428d..9a69d26 100644 --- a/test/copy_outputs_without_meta.yaml +++ b/test/copy_outputs_without_meta.yaml @@ -1,15 +1,20 @@ vcf: -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.annotate-variants.vcf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.annotate-variants.vcf bam: -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.abra.printreads.bam + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_normal.rg.md.abra.printreads.bam maf: -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf -- class: File - path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf \ No newline at end of file + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.foo_normal.muts.maf + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair1/PairWorkflow/bar_tumor.bar_normal.muts.maf +pileup: + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_tumor.rg.md.pileup + - class: File + path: /juno/work/ci/argos-test/data/PairWorkflowPair0/PairWorkflow/foo_normal.rg.md.pileup