From 9c6db8634771ba1293bc248f785d981490cd8ac1 Mon Sep 17 00:00:00 2001 From: dlaehnemann Date: Mon, 6 Nov 2023 14:33:22 +0000 Subject: [PATCH] increase runtimes throughout workflow, to mostly avoid TIMEOUT failures on 1st attempt --- workflow/rules/calling.smk | 10 +++++----- workflow/rules/candidate_calling.smk | 8 ++++---- workflow/rules/mapping.smk | 8 ++++---- workflow/rules/phylogeny.smk | 8 ++++---- workflow/rules/ref.smk | 6 +++--- workflow/rules/regions.smk | 10 +++++----- workflow/rules/trimming.smk | 4 ++-- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/workflow/rules/calling.smk b/workflow/rules/calling.smk index 91a782a..9019da9 100644 --- a/workflow/rules/calling.smk +++ b/workflow/rules/calling.smk @@ -10,7 +10,7 @@ rule scatter_candidate_calls: conda: "../envs/rbt.yaml" resources: - runtime=lambda wildcards, attempt: attempt * 40 - 1, + runtime=lambda wildcards, attempt: attempt * 59 - 1, mem_mb=lambda wildcards, input, attempt: input.size_mb * 1.4 * attempt, shell: "rbt vcf-split {input} {output}" @@ -34,7 +34,7 @@ rule prosolo_calling: extra="", threads: 1 resources: - runtime=lambda wildcards, attempt: 6 * attempt * 60 - 1, + runtime=lambda wildcards, attempt: 8 * attempt * 60 - 1, mem_mb=lambda wildcards, attempt: attempt * 4000, log: "logs/prosolo/{individual}/{sc}.{scatteritem}.merged_bulk.prosolo.log", @@ -53,7 +53,7 @@ rule sort_calls: "../envs/bcftools.yaml" resources: mem_mb=4000, - runtime=lambda wildcards, attempt: 40 * attempt - 1, + runtime=lambda wildcards, attempt: 59 * attempt - 1, shell: "bcftools sort --max-mem {resources.mem_mb}M --temp-dir `mktemp -d` " "-Ob {input} > {output} 2> {log}" @@ -69,7 +69,7 @@ rule bcftools_index_region_calls: conda: "../envs/bcftools.yaml" resources: - runtime=lambda wildcards, attempt: 40 * attempt - 1, + runtime=lambda wildcards, attempt: 59 * attempt - 1, shell: "bcftools index {input} 2> {log}" @@ -89,7 +89,7 @@ rule aggregate_prosolo_chunk_calls: params: extra="-a", resources: - runtime=lambda wildcards, attempt: 40 * attempt - 1, + runtime=lambda wildcards, attempt: 59 * attempt - 1, mem_mb=lambda wildcards, input, attempt: input.size_mb * (1 + 0.5 * attempt) wrapper: "v1.21.1/bio/bcftools/concat" diff --git a/workflow/rules/candidate_calling.smk b/workflow/rules/candidate_calling.smk index f6392d5..fad06e9 100644 --- a/workflow/rules/candidate_calling.smk +++ b/workflow/rules/candidate_calling.smk @@ -50,7 +50,7 @@ rule freebayes_per_region: ), threads: 1 resources: - runtime=lambda wildcards, attempt: 5 * attempt * 60 - 1, + runtime=lambda wildcards, attempt: 7 * attempt * 60 - 1, mem_mb=lambda wc, attempt: 8000 + 16000 * (attempt - 1), shell: "(freebayes {params.extra} -r {wildcards.chromosome}:{wildcards.region} -f {input.ref} {input.samples} | " @@ -67,7 +67,7 @@ rule bcftools_norm_candidate_calls: conda: "../envs/bcftools.yaml" resources: - runtime=lambda wildcards, attempt: 30 * attempt - 1, + runtime=lambda wildcards, attempt: 59 * attempt - 1, shell: # TODO: turn off the following atomize and instead activate --do-not-normalize # once the ProSolo model ist re-integrated with varlociraptor @@ -88,7 +88,7 @@ rule bcftools_index_candidate_calls: params: extra="", # optional parameters for bcftools index resources: - runtime=lambda wildcards, attempt: 20 * attempt - 1, + runtime=lambda wildcards, attempt: 59 * attempt - 1, wrapper: "v1.22.0/bio/bcftools/index" @@ -105,7 +105,7 @@ rule aggregate_freebayes_region_calls: extra="-a", threads: 2 resources: - runtime=lambda wildcards, attempt: 10 * 60 * attempt - 1, + runtime=lambda wildcards, attempt: 12 * 60 * attempt - 1, mem_mb=lambda wildcards, input, attempt: input.size_mb * 6 * (1 + 0.5 * attempt) wrapper: "v1.21.1/bio/bcftools/concat" diff --git a/workflow/rules/mapping.smk b/workflow/rules/mapping.smk index 772bf84..2dfdaae 100644 --- a/workflow/rules/mapping.smk +++ b/workflow/rules/mapping.smk @@ -11,7 +11,7 @@ rule map_reads: sorting="samtools", sort_order="coordinate", resources: - runtime=179, + runtime=60 * 4, mem_mb=16000, threads: 8 wrapper: @@ -42,7 +42,7 @@ rule mark_duplicates: ), resources: mem_mb=4096, - runtime=119, + runtime=60 * 3, # TODO: return to using wrapper, once this PR is propagated to the wrapper: # https://github.com/snakemake/snakemake-wrapper-utils/pull/24 script: @@ -111,7 +111,7 @@ rule apply_bqsr: java_opts="-Xmx3072m", resources: mem_mb=4096, - runtime=89, + runtime=60 * 2, # TODO: return to using wrapper, once this PR is propagated to the wrapper: # https://github.com/snakemake/snakemake-wrapper-utils/pull/24 script: @@ -142,6 +142,6 @@ rule bam_index_merged_bulks: log: "logs/bam_index/merged_bulks/{individual}.merged_bulk.sorted.log", resources: - runtime=lambda wildcards, attempt: 60 * attempt - 1, + runtime=lambda wildcards, attempt: 60 * 2 * attempt - 1, wrapper: "v1.21.1/bio/samtools/index" diff --git a/workflow/rules/phylogeny.smk b/workflow/rules/phylogeny.smk index 6831aff..ad86708 100644 --- a/workflow/rules/phylogeny.smk +++ b/workflow/rules/phylogeny.smk @@ -28,7 +28,7 @@ rule prosolo_probs_to_raxml_ng_ml_gt_and_likelihoods_per_cell: min_prob_genotype=config.get("min_prob_genotype", 0.98) threads: 4 resources: - runtime=lambda wildcards, attempt: attempt * 120 - 1, + runtime=lambda wildcards, attempt: attempt * 60 * 6 - 1, shell: # TODO: do bcftools view filtering for {sc} and coverage in in {sc} before prosolo calling, but # keep it here for now to avoid rerunning already done prosolo calling @@ -78,7 +78,7 @@ rule join_one_more_cell: conda: "../envs/xsv_miller.yaml" resources: - runtime=lambda wildcards, attempt: attempt * 20, + runtime=lambda wildcards, attempt: attempt * 59, # input.size_mb is only queried for the first input file for the group job, so we # need to account for the number of executions of this rule which each adds another # single cell to from this individual @@ -109,7 +109,7 @@ rule parse_to_raxml_ng_gt_and_likelihoods: params: default_likelihoods=",".join(["0.01"] * 10), resources: - runtime=lambda wc, attempt: attempt * 8 * len( get_single_cells_for_individual(wc.individual)), + runtime=lambda wc, attempt: attempt * 15 * len( get_single_cells_for_individual(wc.individual)), mem_mb=lambda wc, attempt, input: attempt * input.size_mb * 2 * len( get_single_cells_for_individual(wc.individual)), threads: 2 shell: @@ -170,7 +170,7 @@ rule raxml_ng_parse: model=config["raxml_ng"].get("model", "GTGTR+FO"), prefix=get_raxml_ng_prefix, resources: - runtime=lambda wildcards, attempt: attempt * 60 - 1, + runtime=lambda wildcards, attempt: attempt * 60 * 2 - 1, mem_mb=lambda wildcards, attempt, input: attempt * 10 * input.size_mb, threads: 2 shell: diff --git a/workflow/rules/ref.smk b/workflow/rules/ref.smk index ad3e89b..a730c20 100644 --- a/workflow/rules/ref.smk +++ b/workflow/rules/ref.smk @@ -56,7 +56,7 @@ rule bwa_index: params: algorithm="bwtsw", resources: - runtime=119, + runtime=lambda wildcards, attempt: 4 * attempt * 60 - 1, mem_mb=9000, wrapper: "v1.21.1/bio/bwa/index" @@ -102,7 +102,7 @@ rule get_known_variants: build=config["ref"]["build"], type="all", resources: - runtime=lambda wc, attempt: attempt * 40 - 1, + runtime=lambda wc, attempt: attempt * 59 - 1, cache: True wrapper: "v1.21.1/bio/reference/ensembl-variation" @@ -116,7 +116,7 @@ rule remove_iupac_codes: log: "logs/rbt/remove_iupac_codes.log", resources: - runtime=lambda wc, attempt: attempt * 80 - 1, + runtime=lambda wc, attempt: attempt * 60 * 3 - 1, conda: "../envs/rbt.yaml" cache: True diff --git a/workflow/rules/regions.smk b/workflow/rules/regions.smk index 8e7b127..3309889 100644 --- a/workflow/rules/regions.smk +++ b/workflow/rules/regions.smk @@ -12,7 +12,7 @@ rule build_sample_regions: extra="--no-per-base", quantize="1:", resources: - runtime=lambda wc, attempt: attempt * 30 - 1, + runtime=lambda wc, attempt: attempt * 59 - 1, wrapper: "v1.21.1/bio/mosdepth" @@ -30,7 +30,7 @@ rule merge_individual_regions: conda: "../envs/bedtools.yaml" resources: - runtime=59, + runtime=lambda wc, attempt: attempt * 59 * 2 - 1, shell: "zcat {input} | sort -k1,1 -k2,2n - | mergeBed -i - -d 15000 > {output} 2> {log}" @@ -69,7 +69,7 @@ rule build_sample_regions_cov: extra="--no-per-base", quantize=f"{config['min_cov_for_candidate_sites']}:", resources: - runtime=lambda wc, attempt: attempt * 30 - 1, + runtime=lambda wc, attempt: attempt * 59 - 1, wrapper: "v1.21.1/bio/mosdepth" @@ -106,7 +106,7 @@ rule merge_individual_cell_regions_cov: conda: "../envs/bedtools.yaml" resources: - runtime=59, + runtime=lambda wc, attempt: 59 * 2 * attempt, shell: "(cat {input.beds} | " " sort -k1,1 -k2,2n - | " @@ -132,7 +132,7 @@ rule merge_individual_bulk_regions_cov: conda: "../envs/bedtools.yaml" resources: - runtime=59, + runtime=lambda wc, attempt: 59 * 2 * attempt, shell: "(cat {input.beds} | " " sort -k1,1 -k2,2n - | " diff --git a/workflow/rules/trimming.smk b/workflow/rules/trimming.smk index 9bad60b..b153577 100644 --- a/workflow/rules/trimming.smk +++ b/workflow/rules/trimming.smk @@ -9,7 +9,7 @@ rule cutadapt: # https://cutadapt.readthedocs.io/en/stable/guide.html#adapter-types adapters=get_cutadapt_parameters, resources: - runtime=lambda wildcards, attempt: 2 * attempt * 60 - 1, + runtime=lambda wildcards, attempt: 4 * attempt * 60 - 1, log: "logs/cutadapt/{sample}.{unit}.log", threads: 4 # set desired number of threads here @@ -25,7 +25,7 @@ rule merge_fastqs: log: "logs/merge-fastqs/{sample}.{read}.log", resources: - runtime=39, + runtime=lambda wildcards, attempt: attempt * 59 - 1, wildcard_constraints: read="1|2", shell: