increase runtimes throughout workflow, to mostly avoid TIMEOUT failur…

…es on 1st attempt

workflow/rules/calling.smk

@@ -10,7 +10,7 @@ rule scatter_candidate_calls:
     conda:
         "../envs/rbt.yaml"
     resources:
-        runtime=lambda wildcards, attempt: attempt * 40 - 1,
+        runtime=lambda wildcards, attempt: attempt * 59 - 1,
         mem_mb=lambda wildcards, input, attempt: input.size_mb * 1.4 * attempt,
     shell:
         "rbt vcf-split {input} {output}"
@@ -34,7 +34,7 @@ rule prosolo_calling:
         extra="",
     threads: 1
     resources:
-        runtime=lambda wildcards, attempt: 6 * attempt * 60 - 1,
+        runtime=lambda wildcards, attempt: 8 * attempt * 60 - 1,
         mem_mb=lambda wildcards, attempt: attempt * 4000,
     log:
         "logs/prosolo/{individual}/{sc}.{scatteritem}.merged_bulk.prosolo.log",
@@ -53,7 +53,7 @@ rule sort_calls:
         "../envs/bcftools.yaml"
     resources:
         mem_mb=4000,
-        runtime=lambda wildcards, attempt: 40 * attempt - 1,
+        runtime=lambda wildcards, attempt: 59 * attempt - 1,
     shell:
         "bcftools sort --max-mem {resources.mem_mb}M --temp-dir `mktemp -d` "
         "-Ob {input} > {output} 2> {log}"
@@ -69,7 +69,7 @@ rule bcftools_index_region_calls:
     conda:
         "../envs/bcftools.yaml"
     resources:
-        runtime=lambda wildcards, attempt: 40 * attempt - 1,
+        runtime=lambda wildcards, attempt: 59 * attempt - 1,
     shell:
         "bcftools index {input} 2> {log}"
 
@@ -89,7 +89,7 @@ rule aggregate_prosolo_chunk_calls:
     params:
         extra="-a",
     resources:
-        runtime=lambda wildcards, attempt: 40 * attempt - 1,
+        runtime=lambda wildcards, attempt: 59 * attempt - 1,
         mem_mb=lambda wildcards, input, attempt: input.size_mb * (1 + 0.5 * attempt)
     wrapper:
         "v1.21.1/bio/bcftools/concat"

workflow/rules/candidate_calling.smk

@@ -50,7 +50,7 @@ rule freebayes_per_region:
         ),
     threads: 1
     resources:
-        runtime=lambda wildcards, attempt: 5 * attempt * 60 - 1,
+        runtime=lambda wildcards, attempt: 7 * attempt * 60 - 1,
         mem_mb=lambda wc, attempt: 8000 + 16000 * (attempt - 1),
     shell:
         "(freebayes {params.extra} -r {wildcards.chromosome}:{wildcards.region} -f {input.ref} {input.samples} | "
@@ -67,7 +67,7 @@ rule bcftools_norm_candidate_calls:
     conda:
         "../envs/bcftools.yaml"
     resources:
-        runtime=lambda wildcards, attempt: 30 * attempt - 1,
+        runtime=lambda wildcards, attempt: 59 * attempt - 1,
     shell:
         # TODO: turn off the following atomize and instead activate --do-not-normalize
         # once the ProSolo model ist re-integrated with varlociraptor
@@ -88,7 +88,7 @@ rule bcftools_index_candidate_calls:
     params:
         extra="",  # optional parameters for bcftools index
     resources:
-        runtime=lambda wildcards, attempt: 20 * attempt - 1,
+        runtime=lambda wildcards, attempt: 59 * attempt - 1,
     wrapper:
         "v1.22.0/bio/bcftools/index"
 
@@ -105,7 +105,7 @@ rule aggregate_freebayes_region_calls:
         extra="-a",
     threads: 2
     resources:
-        runtime=lambda wildcards, attempt: 10 * 60 * attempt - 1,
+        runtime=lambda wildcards, attempt: 12 * 60 * attempt - 1,
         mem_mb=lambda wildcards, input, attempt: input.size_mb * 6 * (1 + 0.5 * attempt)
     wrapper:
         "v1.21.1/bio/bcftools/concat"

workflow/rules/mapping.smk

@@ -11,7 +11,7 @@ rule map_reads:
         sorting="samtools",
         sort_order="coordinate",
     resources:
-        runtime=179,
+        runtime=60 * 4,
         mem_mb=16000,
     threads: 8
     wrapper:
@@ -42,7 +42,7 @@ rule mark_duplicates:
         ),
     resources:
         mem_mb=4096,
-        runtime=119,
+        runtime=60 * 3,
     # TODO: return to using wrapper, once this PR is propagated to the wrapper:
     #   https://github.com/snakemake/snakemake-wrapper-utils/pull/24
     script:
@@ -111,7 +111,7 @@ rule apply_bqsr:
         java_opts="-Xmx3072m",
     resources:
         mem_mb=4096,
-        runtime=89,
+        runtime=60 * 2,
     # TODO: return to using wrapper, once this PR is propagated to the wrapper:
     #   https://github.com/snakemake/snakemake-wrapper-utils/pull/24
     script:
@@ -142,6 +142,6 @@ rule bam_index_merged_bulks:
     log:
         "logs/bam_index/merged_bulks/{individual}.merged_bulk.sorted.log",
     resources:
-        runtime=lambda wildcards, attempt: 60 * attempt - 1,
+        runtime=lambda wildcards, attempt: 60 * 2 * attempt - 1,
     wrapper:
         "v1.21.1/bio/samtools/index"

workflow/rules/phylogeny.smk

@@ -28,7 +28,7 @@ rule prosolo_probs_to_raxml_ng_ml_gt_and_likelihoods_per_cell:
         min_prob_genotype=config.get("min_prob_genotype", 0.98)
     threads: 4
     resources:
-        runtime=lambda wildcards, attempt: attempt * 120 - 1,
+        runtime=lambda wildcards, attempt: attempt * 60 * 6 - 1,
     shell:
         # TODO: do bcftools view filtering for {sc} and coverage in in {sc} before prosolo calling, but
         # keep it here for now to avoid rerunning already done prosolo calling
@@ -78,7 +78,7 @@ rule join_one_more_cell:
     conda:
         "../envs/xsv_miller.yaml"
     resources:
-        runtime=lambda wildcards, attempt: attempt * 20,
+        runtime=lambda wildcards, attempt: attempt * 59,
         # input.size_mb is only queried for the first input file for the group job, so we
         # need to account for the number of executions of this rule which each adds another
         # single cell to from this individual
@@ -109,7 +109,7 @@ rule parse_to_raxml_ng_gt_and_likelihoods:
     params:
         default_likelihoods=",".join(["0.01"] * 10),
     resources:
-        runtime=lambda wc, attempt: attempt * 8 * len( get_single_cells_for_individual(wc.individual)),
+        runtime=lambda wc, attempt: attempt * 15 * len( get_single_cells_for_individual(wc.individual)),
         mem_mb=lambda wc, attempt, input: attempt * input.size_mb * 2 * len( get_single_cells_for_individual(wc.individual)),
     threads: 2
     shell:
@@ -170,7 +170,7 @@ rule raxml_ng_parse:
         model=config["raxml_ng"].get("model", "GTGTR+FO"),
         prefix=get_raxml_ng_prefix,
     resources:
-        runtime=lambda wildcards, attempt: attempt * 60 - 1,
+        runtime=lambda wildcards, attempt: attempt * 60 * 2 - 1,
         mem_mb=lambda wildcards, attempt, input: attempt * 10 * input.size_mb,
     threads: 2
     shell:

workflow/rules/ref.smk

@@ -56,7 +56,7 @@ rule bwa_index:
     params:
         algorithm="bwtsw",
     resources:
-        runtime=119,
+        runtime=lambda wildcards, attempt: 4 * attempt * 60 - 1,
         mem_mb=9000,
     wrapper:
         "v1.21.1/bio/bwa/index"
@@ -102,7 +102,7 @@ rule get_known_variants:
         build=config["ref"]["build"],
         type="all",
     resources:
-        runtime=lambda wc, attempt: attempt * 40 - 1,
+        runtime=lambda wc, attempt: attempt * 59 - 1,
     cache: True
     wrapper:
         "v1.21.1/bio/reference/ensembl-variation"
@@ -116,7 +116,7 @@ rule remove_iupac_codes:
     log:
         "logs/rbt/remove_iupac_codes.log",
     resources:
-        runtime=lambda wc, attempt: attempt * 80 - 1,
+        runtime=lambda wc, attempt: attempt * 60 * 3 - 1,
     conda:
         "../envs/rbt.yaml"
     cache: True

workflow/rules/regions.smk

@@ -12,7 +12,7 @@ rule build_sample_regions:
         extra="--no-per-base",
         quantize="1:",
     resources:
-        runtime=lambda wc, attempt: attempt * 30 - 1,
+        runtime=lambda wc, attempt: attempt * 59 - 1,
     wrapper:
         "v1.21.1/bio/mosdepth"
 
@@ -30,7 +30,7 @@ rule merge_individual_regions:
     conda:
         "../envs/bedtools.yaml"
     resources:
-        runtime=59,
+        runtime=lambda wc, attempt: attempt * 59 * 2 - 1,
     shell:
         "zcat {input} | sort -k1,1 -k2,2n - | mergeBed -i - -d 15000 > {output} 2> {log}"
 
@@ -69,7 +69,7 @@ rule build_sample_regions_cov:
         extra="--no-per-base",
         quantize=f"{config['min_cov_for_candidate_sites']}:",
     resources:
-        runtime=lambda wc, attempt: attempt * 30 - 1,
+        runtime=lambda wc, attempt: attempt * 59 - 1,
     wrapper:
         "v1.21.1/bio/mosdepth"
 
@@ -106,7 +106,7 @@ rule merge_individual_cell_regions_cov:
     conda:
         "../envs/bedtools.yaml"
     resources:
-        runtime=59,
+        runtime=lambda wc, attempt: 59 * 2 * attempt,
     shell:
         "(cat {input.beds} | "
         " sort -k1,1 -k2,2n - | "
@@ -132,7 +132,7 @@ rule merge_individual_bulk_regions_cov:
     conda:
         "../envs/bedtools.yaml"
     resources:
-        runtime=59,
+        runtime=lambda wc, attempt: 59 * 2 * attempt,
     shell:
         "(cat {input.beds} | "
         " sort -k1,1 -k2,2n - | "

workflow/rules/trimming.smk

@@ -9,7 +9,7 @@ rule cutadapt:
         # https://cutadapt.readthedocs.io/en/stable/guide.html#adapter-types
         adapters=get_cutadapt_parameters,
     resources:
-        runtime=lambda wildcards, attempt: 2 * attempt * 60 - 1,
+        runtime=lambda wildcards, attempt: 4 * attempt * 60 - 1,
     log:
         "logs/cutadapt/{sample}.{unit}.log",
     threads: 4  # set desired number of threads here
@@ -25,7 +25,7 @@ rule merge_fastqs:
     log:
         "logs/merge-fastqs/{sample}.{read}.log",
     resources:
-        runtime=39,
+        runtime=lambda wildcards, attempt: attempt * 59 - 1,
     wildcard_constraints:
         read="1|2",
     shell: