feat(assembly): predict megahit memory consumption

Author: pbelmann

docs/modules/assembly.md

@@ -30,10 +30,30 @@
 
 The output is a gzipped fasta file containing contigs.
 
-## Megahit Error Handling
+## Megahit 
+
+### Error Handling
 
 On error with exit codes ([-9, 137]) (e.g. due to memory restrictions), the tool is executed again with higher cpu and memory values.
-The memory and cpu values are computed by the formula 2^(number of attempts) * (cpu/memory value of the assigned flavour).
-The highest possible cpu/memory value is restricted by the highest cpu/memory value of all flavours defined in the resource section 
+The memory and cpu values are computed by the formula 2^(number of attempts) * (cpu/memory value of the assigned or predicted flavor).
+The highest possible cpu/memory value is restricted by the highest cpu/memory value of all flavors defined in the resource section 
 (see global [configuration](../pipeline_configuration.md) section). 
 
+### Peak memory usage prediction
+
+Memory cosumption of an assembler varies based on diversity and size of the dataset. We trained a machine learning model on kmer frequencies
+and the nonpareil diversity index in order to be able to predict the memory peak consumption of megahit in our full pipeline mode. The required
+resources in order to run the assembler are thereby fitted to the resources that are actually needed for a specific dataset. If this
+mode is enabled then Nonpareil and jellyfish that are part of the quality control module are automatically executed before the assembler run.  
+
+```
+  resources:
+    RAM: 
+      mode: MODE
+      predictMinLabel: LABEL
+```
+
+where 
+    * MODE can be either 'PREDICT' for predicting memory usage or 'DEFAULT' for using a default flavor defined in the resources section.
+
+    * LABEL is the flavor that will be used if the predicted RAM is below the memory value defined as part of the LABEL flavor. It can also be set to AUTO to always use the predicted flavor.

docs/pipeline_configuration.md

@@ -241,3 +241,5 @@ resources:
     cpus: 1
     memory: 2
 ```
+
+The full pipeline mode is able to predict the memory consumption of some assemblers (see assembly module section).

example_params/assembly.yml

@@ -11,6 +11,10 @@ steps:
     megahit:
       additionalParams: " --min-contig-len 200 "
       fastg: true
+      resources:
+         RAM: 
+            mode: 'DEFAULT'
+            predictMinLabel: 'AUTO' 
 
 resources:
   large:

example_params/fullPipeline.yml

@@ -15,10 +15,22 @@ steps:
     fastp:
        # Example params: " --cut_front --cut_tail --detect_adapter_for_pe  "
        additionalParams: "  "
+    nonpareil:
+       additionalParams: " -v 10 -r 1234 "
+    jellyfish:
+       additionalParams: 
+         count: " -m 21 -s 100M "
+         histo: " "
+
   assembly:
     megahit:
       additionalParams: " --min-contig-len 200 "
       fastg: true
+      resources:
+        RAM: 
+          mode: 'PREDICT'
+          predictMinLabel: 'medium' 
+
   binning:
     bowtie:
       additionalParams: 

example_params/fullPipeline_fraction/fullPipeline_fraction_binning_metabinner.yml

@@ -18,6 +18,10 @@ steps:
   assembly:
     megahit:
       additionalParams: " --min-contig-len 200 "
+      resources:
+         RAM: 
+            mode: 'DEFAULT'
+            predictMinLabel: 'AUTO' 
   binning:
     bowtie:
       additionalParams: 

example_params/fullPipeline_fraction/fullPipeline_fraction_magAttributes.yml

@@ -18,6 +18,10 @@ steps:
   assembly:
     megahit:
       additionalParams: " --min-contig-len 200 "
+      resources:
+         RAM: 
+            mode: 'DEFAULT'
+            predictMinLabel: 'AUTO' 
   binning:
     bowtie:
       additionalParams: 

example_params/fullPipeline_fraction/fullPipeline_fraction_plasmid.yml

@@ -19,6 +19,10 @@ steps:
     megahit:
       fastg: false
       additionalParams: " --min-contig-len 200 "
+      resources:
+         RAM: 
+            mode: 'DEFAULT'
+            predictMinLabel: 'AUTO' 
   binning:
     bowtie:
       additionalParams: 

main.nf

@@ -187,6 +187,19 @@ workflow _wConfigurePipeline {
 	 assembler, parameter -> params.steps.assembly.get(assembler).putAll(fastg)
        }
     }
+
+    // If memory resources should be predicted by megahit then nonpareil and jellyfish
+    // must be enabled
+    if(params.steps?.assembly?.megahit?.resources?.RAM?.mode == "PREDICT"){
+	if(!params.steps?.qc.containsKey("nonpareil")){
+          def nonpareil = [ nonpareil: [additionalParams: " -v 10 -r 1234 "]]
+          params.steps.qc.putAll(nonpareil) 
+        }
+	if(!params.steps?.qc.containsKey("jellyfish")){
+          def jellyfish = [ jellyfish: [additionalParams: [ count: " -m 21 -s 100M ", histo: " "]]]
+          params.steps.qc.putAll(jellyfish) 
+        }
+    }
 }
 
 
@@ -224,7 +237,7 @@ workflow wPipeline {
     wQualityControlList.out.readsPair \
 	| join(wQualityControlList.out.readsSingle) | set { qcReads }
 
-    wAssemblyList(qcReads)
+    wAssemblyList(qcReads, wQualityControlList.out.nonpareil, wQualityControlList.out.kmerFrequencies)
 
     wBinning(wAssemblyList.out.contigs, qcReads)