1.3 (#765)

* Feature/upgrade delly wf (#753) * Upgrade delly to 0.8.2 and create new Docker containing bcftools and delly together to remove burden from Delly-Manta merging steps * added ps to Docker file and soft-linked delly to /usr/bin * fixed hard-coded reference to delly location, reverted softlink to delly to /usr/local/bin as this was in PATH all along * moved up output of delly in DAG to avoid nextflow failure * fixed publishing of delly calls into results folder * removed originally delly container and insured publish of delly files * including auxiliary Docker files such as ._Dockerfile * removed ._Dockerfile files Co-authored-by: gongyixiao <[email protected]> Co-authored-by: Anne Marie Noronha <[email protected]> * add watchPath * fix bamFiles channel empty * bamMapping.tsv produce absolute path * skip header * standardize aggregate channel names * combine with cohort channel * add watch params * aggregate tag and cohort dir * fix QcBamAggregate combine * add pdf.* in .gitignore * fix comma to semicolon in into * use params.watch and put watchPath in Utils * remove params.watchPath from nextflow.config * remove switches in pipeline.nf * remove switches in pipeline.nf 2 * move watchPath functions into pipeline.nf * remove changes in tempoutils * add support for aggregate cohort tsv * add support for watch aggregate tsv * align * test for aggregate tsv only * support for watch * add switches to aggregate * remove duplciated line * remove duplicated lines2 * add back QcBamAggregate support * Update running-the-pipeline.md * Update running-the-pipeline.md * Update outputs.md * Update outputs.md * Update running-the-pipeline.md * use splitCsv * change cohortSize to be integer * remove duplicates * use splitCsv for watchPath * add pairingQc switch for QcConpair and aggregate * add unique after the groupKey * remove the possibility of duplicate * move RunNeoantigen before SomaticFacetsAnnotation * change max time to 31 days * update containers only * change the max time to 500 hours * fix bug when two final mafs in one germline combined_mutations folder * fix GermlineAggregateSv duplicates * fix index * fix the facets-suite version * install facets-suite * add back install facets * change DoFacets to use wrapper * change summarize_project.py path * facets suite container fix * update annotate-withzygocity scripts * make SomaticFacetsAnnotation work * fix missed change in annotate with zygosity somatic R * remove _em * change output channels for arm gene level * pipeline.nf * change MetaDataParser to be compatible with facets-suite 2.0 * change metadataparser container version in conf * fix SomaticAggregateFacets * make facets output structure compatibale with facets-preview * Revert "move RunNeoantigen before SomaticFacetsAnnotation" * add back changes from PR 758 * docs changes * fix resource genome config * fix wgs different values * fix wgd 2 * fix bugs in QcBamAggregate and Facets * fix 756 * fix bug when _L001_R1_001 and _L001_R1_002 are in fastq files * add check for R1 and R2 order when merging two channels * update container * change geneLevel.R in pipeline.nf * fix typo * add data folder * fix typo * remove mkdir data * change directory * improve splitLanes * initialize pairingQc variable and set pairingQc to true when --QC parameter is indicated * fix facets seed random error (#782) * fix facets seed random error * added retry limit for DoFacets within script chunk to try up to 4 different seeds in one task attempt * fixed typo in DoFacets bash script for calling a bash variable * raise resource usage on DoFacets process to avoid failure due to memory limit * changed the resources for do facets to 1 cpu x 20 Gb mem for juno and aws profiles Co-authored-by: Anne Marie Noronha <[email protected]> * fixed bam truncation bug by using beforeScript and catch_USR2 (#767) * fixed bam truncation bug by using beforeScript and catch_USR2 * move beforeScript into juno.config to fix travis * fix close brace after beforeScript * a working solution for catch USR2 * Bugfix/final touchup1.3 (#790) * double Neoantigen retry memory increase * disable lohhla by default * docs revise * added similar note on disabling of lohhla to "bioinformatic-components.md" * fixed redundant text in "variant-annotation-and-filtering.md" Co-authored-by: Anne Marie Noronha <[email protected]> Co-authored-by: anoronh4 <[email protected]> Co-authored-by: anoronh4 <[email protected]> Co-authored-by: Anne Marie Noronha <[email protected]>
mskcc · May 7, 2020 · 6d3990f · 6d3990f
1 parent 94e134a
commit 6d3990f
Show file tree

Hide file tree

Showing 34 changed files with 822 additions and 478 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,7 @@ nf.*
 *.txt
 *.txt.*
 *.pdf
+*.pdf.*
 *.gz
 *.bam.*
 Result/

diff --git a/conf/containers.config b/conf/containers.config
@@ -31,7 +31,7 @@
 //------------------- Somatic pipeline
 
   withName:SomaticDellyCall {
-    container = "cmopipeline/delly:v0.8.1"
+    container = "cmopipeline/delly-bcftools:0.0.1"
   }
   withName:RunMutect2 {
     container = "broadinstitute/gatk:4.1.0.0"
@@ -49,13 +49,13 @@
     container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.0"
   }
   withName:SomaticCombineChannel {
-    container = "cmopipeline/bcftools-vt:1.2.0"
+    container = "cmopipeline/bcftools-vt:1.2.1"
   }
   withName:SomaticAnnotateMaf {
     container = "cmopipeline/vcf2maf:vep88_1.2.3"
   }
   withName:DoFacets {
-    container = "cmopipeline/facets-htstools:2.3.0"
+    container = "cmopipeline/facets-htstools:3.0.1"
   } 
   withName:RunMsiSensor {
     container = "vanallenlab/msisensor:0.5"
@@ -70,20 +70,20 @@
     container = "cmopipeline/mutation-signatures:1.1.0"
   }
   withName:SomaticFacetsAnnotation {
-    container = "cmopipeline/facets-htstools:2.3.0"
+    container = "cmopipeline/facets-htstools:3.0.1"
   } 
   withName:RunNeoantigen {
     container = "cmopipeline/neoantigen:0.3.1"
   }
   withName:MetaDataParser {
-    container = "cmopipeline/metadataparser:0.5.1"
+    container = "cmopipeline/metadataparser:0.5.2"
   }
 
 
 //------------------- Germline pipeline
 
   withName:GermlineDellyCall {
-    container = "cmopipeline/delly:v0.8.1"
+    container = "cmopipeline/delly-bcftools:0.0.1"
   }
   withName:GermlineRunHaplotypecaller {
     container = "broadinstitute/gatk:4.1.0.0"
@@ -98,13 +98,13 @@
     container = "cmopipeline/strelka2_manta:latest"
   }
   withName:GermlineCombineChannel {
-    container = "cmopipeline/bcftools-vt:1.2.0"
+    container = "cmopipeline/bcftools-vt:1.2.1"
   }
   withName:GermlineAnnotateMaf {
     container = "cmopipeline/vcf2maf:vep88_1.2.3"
   }
   withName:GermlineFacetsAnnotation {
-    container = "cmopipeline/facets-htstools:2.3.0"
+    container = "cmopipeline/facets-htstools:3.0.1"
   } 
   withName:GermlineMergeDellyAndManta {
     container = "cmopipeline/bcftools-vt:1.1.1"
@@ -155,7 +155,7 @@
     container = "cmopipeline/bcftools-vt:1.2.0"
   }
   withName:QcBamAggregate {
-    container = "cmopipeline/metadataparser:0.5.1"
+    container = "cmopipeline/metadataparser:0.5.2"
   }
   withName:QcConpairAggregate {
     container = "cmopipeline/conpair:v0.3.3"

diff --git a/conf/exome.config b/conf/exome.config
@@ -13,12 +13,10 @@ params {
     ndepth = 35
     min_nhet = 25
     purity_cval = 500
-    purity_snp_nbhd = 250
-    purity_ndepth = 35
     purity_min_nhet = 25
     genome = "hg19"
     R_lib = "0.5.14"
-    seed = 1000
+    seed = 100
   }
   somaticVariant {
     tumorVaf = 0.05

diff --git a/conf/genome.config b/conf/genome.config
@@ -13,12 +13,10 @@ params {
     ndepth = 15
     min_nhet = 25
     purity_cval = 5000
-    purity_snp_nbhd = 500
-    purity_ndepth = 15
     purity_min_nhet = 25
     genome = "hg19"
     R_lib = "0.5.14"
-    seed = 1000
+    seed = 100
   }
   somaticVariant {
     tumorVaf = 0.05

diff --git a/conf/juno.config b/conf/juno.config
@@ -14,10 +14,10 @@ executor {
 
 process {
   memory = "8.GB"
-  time = { 3.h * task.attempt }
+  time = { task.attempt < 3 ? 3.h * task.attempt  : 500.h }
   clusterOptions = ""
   scratch = true
-  beforeScript = "module load singularity/3.1.1; unset R_LIBS"
+  beforeScript = "module load singularity/3.1.1; unset R_LIBS; catch_USR2 () { echo 'caught USR2 signal'; set +e; false; on_exit ; } ; trap catch_USR2 USR2"
 }
 
 process.errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }

diff --git a/conf/resources.config b/conf/resources.config
diff --git a/conf/resources_aws.config b/conf/resources_aws.config
@@ -82,7 +82,7 @@
   }
   withName:DoFacets {
     cpus = { 1 }
-    memory = { 10.GB * task.attempt }
+    memory = { 20.GB * task.attempt }
   }
   withName:RunMsiSensor {
     cpus = { 1 }
@@ -102,7 +102,7 @@
   }
   withName:RunNeoantigen {
     cpus = { 1 }
-    memory = { 8.GB * task.attempt }
+    memory = { 8.GB * task.attempt * 2 }
   }
   withName:MetaDataParser {
     cpus = { 1 }

diff --git a/conf/resources_aws_genome.config b/conf/resources_aws_genome.config
@@ -102,7 +102,7 @@
   }
   withName:RunNeoantigen {
     cpus = { 1 }
-    memory = { 8.GB * task.attempt }
+    memory = { 8.GB * task.attempt * 2 }
   }
   withName:MetaDataParser {
     cpus = { 1 }

diff --git a/conf/resources_juno.config b/conf/resources_juno.config
@@ -47,12 +47,10 @@
   withName:SomaticDellyCall {
     cpus = { 1 }
     memory = { 8.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:RunMutect2 {
     cpus = { 1 }
     memory = { 4.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:SomaticCombineMutect2Vcf {
     cpus = { 1 }
@@ -61,12 +59,10 @@
   withName:SomaticRunManta {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:SomaticRunStrelka2 {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:SomaticCombineChannel {
     cpus = { 1 }
@@ -82,22 +78,19 @@
   }
   withName:DoFacets {
     cpus = { 1 }
-    memory = { 10.GB * task.attempt }
+    memory = { 20.GB * task.attempt }
   }
   withName:RunMsiSensor {
     cpus = { 1 }
     memory = { 2.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:RunPolysolver {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:RunLOHHLA {
     cpus = { 1 }
     memory = { 3.GB * task.attempt }
-    time = { 3.h * task.attempt }
   } 
   withName:RunMutationSignatures {
     cpus = { 1 }
@@ -109,8 +102,7 @@
   }
   withName:RunNeoantigen {
     cpus = { 1 }
-    memory = { 8.GB * task.attempt }
-    time = { 3.h * task.attempt }
+    memory = { 8.GB * task.attempt * 2 }
   }
   withName:MetaDataParser {
     cpus = { 1 }

diff --git a/conf/resources_juno_genome.config b/conf/resources_juno_genome.config
@@ -47,12 +47,11 @@
   withName:SomaticDellyCall {
     cpus = { 1 }
     memory = { 16.GB * task.attempt }
-    time = { 72.h * task.attempt }
+    time = { 500.h }
   }
   withName:RunMutect2 {
     cpus = { 1 }
     memory = { 4.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:SomaticCombineMutect2Vcf {
     cpus = { 1 }
@@ -61,17 +60,16 @@
   withName:SomaticRunManta {
     cpus = { 8 }
     memory = { 2.GB * task.attempt }
-    time = { 6.h * task.attempt }
+    time = { task.attempt < 2 ? 6.h : 500.h }
   }
   withName:SomaticRunStrelka2 {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 6.h * task.attempt }
+    time = { task.attempt < 2 ? 6.h : 500.h }
   }
   withName:SomaticCombineChannel {
     cpus = { 1 }
     memory = { 4.GB * task.attempt } 
-    time = { 3.h * task.attempt }
   }
   withName:SomaticAnnotateMaf {
     cpus = { 1 }
@@ -88,31 +86,30 @@
   withName:DoFacets {
     cpus = { 1 }
     memory = { 40.GB * task.attempt }
-    time = { 6.h * task.attempt }
+    time = { task.attempt < 2 ? 6.h : 500.h }
   }
   withName:RunMsiSensor {
     cpus = { 1 }
     memory = { 6.GB * task.attempt }
-    time = { 72.h * task.attempt }
+    time = { 500.h }
   }
   withName:RunPolysolver {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 3.h * task.attempt }
   }
   withName:RunLOHHLA {
     cpus = { 1 }
     memory = { 3.GB * task.attempt }
-    time = { 72.h * task.attempt }
+    time = { 500.h }
   } 
   withName:RunMutationSignatures {
     cpus = { 1 }
     memory = { 1.GB * task.attempt }
   }
   withName:RunNeoantigen {
     cpus = { 1 }
-    memory = { 8.GB * task.attempt }
-    time = { 6.h * task.attempt }
+    memory = { 8.GB * task.attempt * 2 }
+    time = { task.attempt < 2 ? 6.h : 500.h }
   }
   withName:MetaDataParser {
     cpus = { 1 }
@@ -124,7 +121,7 @@
   withName:GermlineDellyCall {
     cpus = { 1 }
     memory = { 16.GB * task.attempt }
-    time = { 72.h * task.attempt }
+    time = { 500.h }
   }
   withName:GermlineRunHaplotypecaller {
     cpus = { 2 }
@@ -137,17 +134,17 @@
   withName:GermlineRunManta {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 72.h * task.attempt }
+    time = { 500.h }
   }
   withName:GermlineRunStrelka2 {
     cpus = { 8 }
     memory = { 1.GB * task.attempt }
-    time = { 6.h * task.attempt }
+    time = { task.attempt < 2 ? 6.h : 500.h }
   }
   withName:GermlineCombineChannel {
     cpus = { 1 }
     memory = { 4.GB * task.attempt }
-    time = { 6.h * task.attempt }
+    time = { task.attempt < 2 ? 6.h : 500.h }
   }
   withName:GermlineAnnotateMaf {
     cpus = { 1 }

diff --git a/containers/bcftools-vt/Dockerfile b/containers/bcftools-vt/Dockerfile
@@ -2,9 +2,9 @@ FROM halllab/bcftools:v1.9
 
 LABEL maintainer="C. Allan Bolipata ([email protected])" \
     contributor="Philip Jonsson ([email protected])" \
-    version.image="1.2.0" \
+    version.image="1.2.1" \
     version.vt="0.57721" \
-    version.filter-vcf="0.2.0" \
+    version.filter-vcf="0.2.1" \
     version.pysam="0.15.2" \
     source.getBaseCountsMultiSample="https://github.com/zengzheng123/GetBaseCountsMultiSample/releases/tag/v1.2.2" \
     version.getBaseCountsMultiSample="1.2.2"

diff --git a/containers/bcftools-vt/filter-vcf.py b/containers/bcftools-vt/filter-vcf.py
@@ -8,8 +8,9 @@
 """
 
 __author__  = "Philip Jonsson"
-__email__   = "[email protected]"
-__version__ = "0.2.0"
+__contributor__  = "Yixiao Gong"
+__email__   = "[email protected]; [email protected]"
+__version__ = "0.2.1"
 __status__  = "Dev"
 
 import sys, os
@@ -75,13 +76,14 @@
             'C': var.samples[tumor]['CU'],
             'G': var.samples[tumor]['GU'],
             'T': var.samples[tumor]['TU']
-        }        
-        tier1_other = sum([alleles[a][0] for a in alleles if a not in [alt, ref]])
-        tier2_other = sum([alleles[a][1] for a in alleles if a not in [alt, ref]])
-        alt_reads = alleles[alt][0]
-
-        if (tier1_other + tier2_other) >= alt_reads * 0.5:
-            new_flags.append('multiallelic2')
+        }
+	valid_alleles = [key for key in alleles.keys() if all(isinstance(item,int) for item in alleles[key])]
+        tier1_other = sum([alleles[a][0] for a in valid_alleles if a not in [alt, ref]])
+        tier2_other = sum([alleles[a][1] for a in valid_alleles if a not in [alt, ref]])
+	if alt in valid_alleles:
+		alt_reads = alleles[alt][0]
+		if (tier1_other + tier2_other) >= alt_reads * 0.5:
+			new_flags.append('multiallelic2')
 
     ## Add an additional strand-bias filer
     ## Only MuTect2 provides sufficient variant information for this

diff --git a/containers/delly/Dockerfile → containers/delly-bcftools/Dockerfile b/containers/delly/Dockerfile → containers/delly-bcftools/Dockerfile
@@ -1,12 +1,12 @@
-FROM ubuntu:18.04
+FROM halllab/bcftools:v1.9
 
-LABEL maintainer="Sinisa Ivkovic (ivkovics@mskcc.org)" \
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
       version.ubuntu="18.04" \
-      version.delly="0.8.1" \
+      version.delly="0.8.2" \
       version.htslib="1.9"
 
 # set the environment variables
-ENV delly_version 0.8.1
+ENV delly_version 0.8.2
 ENV htslib_version 1.9
 
 # run update
@@ -17,6 +17,7 @@ RUN apt-get update -y && apt-get install -y \
     vim  \
     wget \
     unzip \
+    procps \
     build-essential \
     libboost-all-dev \
     zlib1g-dev \
@@ -45,4 +46,4 @@ RUN unzip v${delly_version}.zip
 WORKDIR /usr/local/bin/delly-${delly_version}
 RUN make all
 WORKDIR /usr/local/bin
-RUN ln -s /usr/local/bin/delly-0.8.1/src/delly /usr/local/bin/delly
+RUN ln -s /usr/local/bin/delly-${delly_version}/src/delly /usr/local/bin/delly
-Original file line number
+Diff line change
@@ Expand Up / @@ -15,6 +15,7 @@ nf.* @@
     *.txt
     *.txt.*
     *.pdf
+    *.pdf.*
     *.gz
     *.bam.*
     Result/
@@ Expand Down @@