updated vadr and nextclade processes (#70)

* update formatting

* spelling error

* Update cecret.nf

* Update cecret_config_template.config

* Added samtools_depth

* Added samtools depth

* Updated summary file for pangolin and nextclade changes

Pangolin and Nextclade both updated, so this has as well. 
Pangolin's scorpio is now a column in the summary file
process combine_summary was removed because it's not needed anymore

* Update cecret_annotation.nf

* simplified vadr variables

* simplified vadr variables

* Now it should be perfect

* Update cecret.nf

* Update Update

* Changed nextclade container

vadr and nextclade are external tools that went through some significant changes that needed to be addressed

* Updated vadr and nextclade

* Updated vadr and nextclade

* updated nextclade container

Author: erinyoung

staphb_toolkit/workflows/cecret/cecret.nf

@@ -3,7 +3,7 @@
 println("Currently using the Cecret workflow for use with amplicon-based Illumina hybrid library prep on MiSeq\n")
 println("Author: Erin Young")
 println("email: [email protected]")
-println("Version: v.20210611")
+println("Version: v.20210815")
 println("")
 
 params.reads = workflow.launchDir + '/reads'
@@ -642,7 +642,8 @@ process ivar_consensus {
   set val(sample), file(bam), file(reference_genome) from trimmed_bams_ivar_consensus
 
   output:
-  tuple sample, file("consensus/${sample}.consensus.fa") into consensus_nextclade, consensus_rename, consensus_pangolin, consensus_vadr
+  tuple sample, file("consensus/${sample}.consensus.fa") into consensus_rename, consensus_pangolin, consensus_vadr
+  tuple sample, file("consensus/${sample}.consensus.fa"), file(reference_genome) into consensus_nextclade
   file("consensus/${sample}.consensus.fa") into consensus_mafft
   file("logs/${task.process}/${sample}.${workflow.sessionId}.{log,err}")
   tuple sample, env(num_N), env(num_ACTG), env(num_degenerate), env(num_total) into consensus_results
@@ -1191,44 +1192,59 @@ pangolin_files
     storeDir: "${params.outdir}/pangolin")
 
 params.nextclade_options = ''
+params.nextclade_genes = 'E,M,N,ORF1a,ORF1b,ORF3a,ORF6,ORF7a,ORF7b,ORF8,ORF9b,S'
 process nextclade {
   publishDir "${params.outdir}", mode: 'copy'
   tag "${sample}"
   echo false
   cpus params.medcpus
-  container 'neherlab/nextclade:latest'
+  container 'nextstrain/nextclade:latest'
 
   when:
   params.nextclade
 
   input:
-  set val(sample), file(fasta) from consensus_nextclade
+  tuple val(sample), file(fasta), file(reference) from consensus_nextclade
 
   output:
-  file("${task.process}/${sample}_nextclade_report.csv") into nextclade_files
+  file("${task.process}/${sample}/${sample}_nextclade.csv") into nextclade_files
   tuple sample, env(nextclade_clade) into nextclade_clade_results
   file("logs/${task.process}/${sample}.${workflow.sessionId}.{log,err}")
 
   shell:
   '''
-    mkdir -p !{task.process} logs/!{task.process}
+    mkdir -p !{task.process}/!{sample} logs/!{task.process}
     log_file=logs/!{task.process}/!{sample}.!{workflow.sessionId}.log
     err_file=logs/!{task.process}/!{sample}.!{workflow.sessionId}.err
 
     date | tee -a $log_file $err_file > /dev/null
     nextclade --version >> $log_file
 
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/genemap.gff
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/tree.json
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/qc.json
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/primers.csv
+
     nextclade !{params.nextclade_options} \
-      --jobs !{task.cpus} \
-      --input-fasta !{fasta} \
-      --output-csv !{task.process}/!{sample}_nextclade_report.csv \
+      --input-fasta=!{fasta} \
+      --input-root-seq=!{reference} \
+      --genes=!{params.nextclade_genes} \
+      --input-gene-map=genemap.gff \
+      --input-tree=tree.json \
+      --input-qc-config=qc.json \
+      --input-pcr-primers=primers.csv \
+      --output-json=!{task.process}/!{sample}/!{sample}_nextclade.json \
+      --output-csv=!{task.process}/!{sample}/!{sample}_nextclade.csv \
+      --output-tsv=!{task.process}/!{sample}/!{sample}_nextclade.tsv \
+      --output-tree=!{task.process}/!{sample}/!{sample}_nextclade.auspice.json \
+      --output-dir=!{task.process}/!{sample} \
+      --output-basename=!{sample} \
       2>> $err_file >> $log_file
 
-
-    nextclade_column=$(head -n 1 !{task.process}/!{sample}_nextclade_report.csv | tr ';' '\\n' | grep -wn "clade" | cut -f 1 -d ":" )
+    nextclade_column=$(head -n 1 !{task.process}/!{sample}/!{sample}_nextclade.csv | tr ';' '\\n' | grep -wn "clade" | cut -f 1 -d ":" )
     if [ -n "$nextclade_column" ]
     then
-      nextclade_clade=$(cat !{task.process}/!{sample}_nextclade_report.csv | grep !{sample} | cut -f $nextclade_column -d ";" | sed 's/,/;/g' | head -n 1 )
+      nextclade_clade=$(cat !{task.process}/!{sample}/!{sample}_nextclade.csv | grep !{sample} | cut -f $nextclade_column -d ";" | sed 's/,/;/g' | sed 's/"//g' | head -n 1 )
     else
       nextclade_clade="Not Found"
     fi
@@ -1243,23 +1259,14 @@ nextclade_files
     sort: true,
     storeDir: "${params.outdir}/nextclade")
 
-if ( Math.round(Runtime.runtime.totalMemory() / 10241024) / 2 > params.medcpus && params.vadr ) {
-  vadrmemory = params.medcpus + params.medcpus
-  vadrcpus = params.medcpus
-} else {
-  vadrmemory = 2
-  vadrcpus = 1
-}
-
-params.vadr_options = '--split --glsearch -s  -r --nomisc --lowsim5term 2 --lowsim3term 2 --alt_fail lowscore,fstukcnf,insertnn,deletinn'
+params.vadr_options = '--split --glsearch -s -r --nomisc --mkey sarscov2 --lowsim5seq 6 --lowsim3seq 6 --alt_fail lowscore,insertnn,deletinn'
 params.vadr_reference = 'sarscov2'
 params.vadr_mdir = '/opt/vadr/vadr-models'
 process vadr {
   publishDir "${params.outdir}", mode: 'copy'
   tag "${sample}"
   echo false
-  cpus vadrcpus
-  memory vadrmemory.GB
+  cpus params.medcpus
   container 'staphb/vadr:latest'
 
   when:
@@ -1289,10 +1296,10 @@ process vadr {
     v-annotate.pl -h >> $log_file
 
     v-annotate.pl !{params.vadr_options} \
+      --cpu !{task.cpus} \
       --noseqnamemax \
       --mkey !{params.vadr_reference} \
       --mdir !{params.vadr_mdir} \
-      --cpu !{task.cpus} \
       !{fasta} \
       !{task.process}/!{sample} \
       2>> $err_file >> $log_file
@@ -1656,3 +1663,4 @@ workflow.onComplete {
     println("A summary of results can be found in a tab-delimited file: ${workflow.launchDir}/run_results.txt")
     println("Execution status: ${ workflow.success ? 'OK' : 'failed' }")
 }
+

staphb_toolkit/workflows/cecret/cecret_annotation.nf

@@ -3,7 +3,7 @@
 println("For annotating SARS-CoV-2 fastas with pangolin, nextclade, and vadr\n")
 println("Author: Erin Young")
 println("email: [email protected]")
-println("Version: v.0.20210611")
+println("Version: v.0.20210815")
 println("")
 
 params.fastas = workflow.launchDir + '/fastas'
@@ -28,6 +28,19 @@ Channel
   }
   .set { fastas }
 
+params.vadr = true
+params.pangolin = true
+params.relatedness = false
+params.nextclade = true
+if (params.nextclade) {
+  Channel
+    .fromPath(params.reference_genome, type:'file')
+    .ifEmpty{
+      println("No reference genome was selected. Set with 'params.reference_genome'")
+    }
+    .set { reference_genome_nextclade }
+}
+
 process fasta_prep {
   publishDir "${params.outdir}", mode: 'copy', overwrite: true
   tag "${fasta}"
@@ -39,7 +52,7 @@ process fasta_prep {
   file(fasta) from fastas
 
   output:
-  file("${task.process}/${fasta}") into fastas_mafft, fastas_pangolin, fastas_nextclade, fastas_vadr
+  file("${task.process}/${fasta}") into prepped_fastas, fastas_mafft, fastas_pangolin, fastas_nextclade, fastas_vadr
 
   shell:
   '''
@@ -54,7 +67,10 @@ process fasta_prep {
   '''
 }
 
-params.pangolin = true
+prepped_fastas
+  .collectFile(name: "Ultimate.fasta", storeDir: "${params.outdir}")
+  .into { multifasta_pangolin ; multifasta_vadr ; multifasta_nextclade ; multifasta_mafft }
+
 params.pangolin_options = ''
 process pangolin {
   publishDir "${params.outdir}", mode: 'copy'
@@ -67,11 +83,10 @@ process pangolin {
   params.pangolin
 
   input:
-  file(fasta) from fastas_pangolin.collect()
+  file(fasta) from multifasta_pangolin
 
   output:
   file("${task.process}/lineage_report.csv")
-  file("${task.process}/ultimate.fasta")
   file("logs/${task.process}/${workflow.sessionId}.{log,err}")
 
   shell:
@@ -83,33 +98,32 @@ process pangolin {
     date | tee -a $log_file $err_file > /dev/null
     pangolin --version >> $log_file
 
-    cat !{fasta} > !{task.process}/ultimate.fasta
-
     pangolin !{params.pangolin_options} \
       --outdir !{task.process} \
-      !{task.process}/ultimate.fasta \
+      !{fasta} \
       2>> $err_file >> $log_file
   '''
 }
 
-params.nextclade = true
 params.nextclade_options = ''
+params.nextclade_genes = 'E,M,N,ORF1a,ORF1b,ORF3a,ORF6,ORF7a,ORF7b,ORF8,ORF9b,S'
 process nextclade {
   publishDir "${params.outdir}", mode: 'copy'
   tag "Clade assignment with nextclade"
   echo false
   cpus params.medcpus
-  container 'neherlab/nextclade:latest'
+  //container 'docker://quay.io/biocontainers/nextclade:1.2.0--h9ee0642_0'
+  container 'nextstrain/nextclade:latest'
 
   when:
   params.nextclade
 
   input:
-  file(fasta) from fastas_nextclade.collect()
+  file(fasta) from multifasta_nextclade
+  file(reference) from reference_genome_nextclade
 
   output:
-  file("${task.process}/nextclade_report.tsv")
-  file("${task.process}/ultimate.fasta")
+  file("${task.process}/*")
   file("logs/${task.process}/${workflow.sessionId}.{log,err}")
 
   shell:
@@ -121,42 +135,45 @@ process nextclade {
     date | tee -a $log_file $err_file > /dev/null
     nextclade --version >> $log_file
 
-    cat !{fasta} > !{task.process}/ultimate.fasta
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/genemap.gff
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/tree.json
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/qc.json
+    wget --no-check-certificate https://raw.githubusercontent.com/nextstrain/nextclade/master/data/sars-cov-2/primers.csv
 
     nextclade !{params.nextclade_options} \
-      --jobs !{task.cpus} \
-      --input-fasta !{task.process}/ultimate.fasta \
-      --output-tsv !{task.process}/nextclade_report.tsv \
+      --input-fasta=!{fasta} \
+      --input-root-seq=!{reference} \
+      --genes=!{params.nextclade_genes} \
+      --input-gene-map=genemap.gff \
+      --input-tree=tree.json \
+      --input-qc-config=qc.json \
+      --input-pcr-primers=primers.csv \
+      --output-json=!{task.process}/nextclade.json \
+      --output-csv=!{task.process}/nextclade.csv \
+      --output-tsv=!{task.process}/nextclade.tsv \
+      --output-tree=!{task.process}/nextclade.auspice.json \
+      --output-dir=!{task.process} \
+      --output-basename=!{task.process} \
       2>> $err_file >> $log_file
   '''
 }
 
-params.vadr = true
-if ( Math.round(Runtime.runtime.totalMemory() / 10241024) / 2 > params.medcpus && params.vadr ) {
-  vadrmemory = params.medcpus + params.medcpus
-  vadrcpus = params.medcpus
-} else {
-  vadrmemory = 2
-  vadrcpus = 1
-}
-
-params.vadr_options = '--split --glsearch -s  -r --nomisc --lowsim5term 2 --lowsim3term 2 --alt_fail lowscore,fstukcnf,insertnn,deletinn'
+params.vadr_options = '--split --glsearch -s -r --nomisc --mkey sarscov2 --lowsim5seq 6 --lowsim3seq 6 --alt_fail lowscore,insertnn,deletinn'
 params.vadr_reference = 'sarscov2'
 params.vadr_mdir = '/opt/vadr/vadr-models'
 process vadr {
   publishDir "${params.outdir}", mode: 'copy'
   tag "Fasta QC with vadr"
   echo false
-  cpus vadrcpus
-  memory vadrmemory.GB
+  cpus params.medcpus
   container 'staphb/vadr:latest'
-  stageInMode = 'symlink'
+  //stageInMode = 'symlink'
 
   when:
   params.vadr
 
   input:
-  file(fasta) from fastas_vadr.collect()
+  file(fasta) from multifasta_vadr
 
   output:
   file("${task.process}/*")
@@ -172,20 +189,17 @@ process vadr {
     echo "no version" >> $log_file
     v-annotate.pl -h >> $log_file
 
-    cat !{fasta} > ultimate.fasta
-
     v-annotate.pl !{params.vadr_options} \
+      --cpu !{task.cpus} \
       --noseqnamemax \
       --mkey !{params.vadr_reference} \
       --mdir !{params.vadr_mdir} \
-      --cpu !{task.cpus} \
-      ultimate.fasta \
+      !{fasta} \
       !{task.process} \
       2>> $err_file >> $log_file
   '''
 }
 
-params.relatedness = false
 if (params.relatedness){
   Channel
     .fromPath(params.reference_genome, type:'file')
@@ -205,7 +219,7 @@ if (params.relatedness){
     maxRetries 3
 
     input:
-    file(fasta) from fastas_mafft.collect()
+    file(fasta) from multifasta_mafft
     file(reference) from reference_genome
 
     output:
@@ -229,12 +243,11 @@ if (params.relatedness){
       echo ">!{params.outgroup}" > reference.fasta
       grep -v ">" !{reference} >> reference.fasta
 
-      cat !{fasta} > !{task.process}/ultimate.fasta
       mafft !{params.mafft_options} \
         --auto \
         --thread !{task.cpus} \
         --maxambiguous !{params.max_ambiguous} \
-        --addfragments !{task.process}/ultimate.fasta \
+        --addfragments !{fasta} \
         reference.fasta \
         > !{task.process}/mafft_aligned.fasta \
         2>> $err_file

staphb_toolkit/workflows/cecret/configs/cecret_config_template.config

@@ -67,7 +67,7 @@ bamsnap_container = 'danielmsk/bamsnap:latest'
 kraken2_container = 'staphb/kraken2:latest'
 bedtools_container = 'staphb/bedtools:latest'
 pangolin_container = 'staphb/pangolin:latest'
-nextclade_container = 'neherlab/nextclade:latest'
+nextclade_container = 'nextstrain/nextclade:latest'
 vadr_container = 'staphb/vadr:latest'
 parallel_perl_container = 'staphb/parallel-perl:latest'
 mafft_container = 'staphb/mafft:latest'
@@ -151,10 +151,11 @@ iqtree_container = 'staphb/iqtree:latest'
 
 //# For process nextclade
 //params.nextclade_options = ''
+//params.nextclade_genes = 'E,M,N,ORF1a,ORF1b,ORF3a,ORF6,ORF7a,ORF7b,ORF8,ORF9b,S'
 //params.nextclade = true
 
 //# For process vadr
-//params.vadr_options = '--split --glsearch -s  -r --nomisc --lowsim5term 2 --lowsim3term 2 --alt_fail lowscore,fstukcnf,insertnn,deletinn'
+//params.vadr_options = '--split --glsearch -s -r --nomisc --mkey sarscov2 --lowsim5seq 6 --lowsim3seq 6 --alt_fail lowscore,insertnn,deletinn'
 //params.vadr_reference = 'sarscov2'
 //params.vadr_mdir = '/opt/vadr/vadr-models'
 //params.vadr = true

staphb_toolkit/workflows/cecret/configs/docker_containers.config

@@ -14,6 +14,6 @@ bcftools_container = 'staphb/bcftools:latest'
 kraken2_container = 'staphb/kraken2:latest'
 bedtools_container = 'staphb/bedtools:latest'
 pangolin_container = 'staphb/pangolin:latest'
-nextclade_container = 'neherlab/nextclade:latest'
+nextclade_container = 'nextstrain/nextclade:latest'
 lightweight_container = 'staphb/parallel-perl:latest'
 iqtree_container = 'staphb/iqtree:latest'