From 8931566ab918a24364b97b17857a60b47cef4942 Mon Sep 17 00:00:00 2001 From: Thiseas Christos Lamnidis Date: Tue, 30 Nov 2021 15:15:40 +0100 Subject: [PATCH 01/16] Version-bump --- .github/workflows/ci.yml | 4 ++-- CHANGELOG.md | 10 ++++++++++ Dockerfile | 4 ++-- environment.yml | 2 +- nextflow.config | 4 ++-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70e0ec0ac..21b7ae020 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,13 +37,13 @@ jobs: - name: Build new docker image if: env.MATCHED_FILES - run: docker build --no-cache . -t nfcore/eager:2.4.1 + run: docker build --no-cache . -t nfcore/eager:dev - name: Pull docker image if: ${{ !env.MATCHED_FILES }} run: | docker pull nfcore/eager:dev - docker tag nfcore/eager:dev nfcore/eager:2.4.1 + docker tag nfcore/eager:dev nfcore/eager:dev - name: Install Nextflow env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d4adf35..144b48f91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [dev] + +### `Added` + +### `Fixed` + +### `Dependencies` + +### `Deprecated` + ## [2.4.1] - 2021-11-30 ### `Added` diff --git a/Dockerfile b/Dockerfile index 1354408d3..cec616444 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY environment.yml / RUN conda env create --quiet -f /environment.yml && conda clean -a # Add conda installation dir to PATH (instead of doing 'conda activate') -ENV PATH /opt/conda/envs/nf-core-eager-2.4.1/bin:$PATH +ENV PATH /opt/conda/envs/nf-core-eager-2.4.2dev/bin:$PATH # Dump the details of the installed packages to a file for posterity -RUN conda env export --name nf-core-eager-2.4.1 > nf-core-eager-2.4.1.yml \ No newline at end of file +RUN conda env export --name nf-core-eager-2.4.2dev > nf-core-eager-2.4.2dev.yml \ No newline at end of file diff --git a/environment.yml b/environment.yml index db6c11b10..b4b3365cb 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,6 @@ # You can use this file to create a conda environment for this pipeline: # conda env create -f environment.yml -name: nf-core-eager-2.4.1 +name: nf-core-eager-2.4.2dev channels: - conda-forge - bioconda diff --git a/nextflow.config b/nextflow.config index 20076e27e..1bddbb339 100644 --- a/nextflow.config +++ b/nextflow.config @@ -284,7 +284,7 @@ params { // Container slug. Stable releases should specify release tag! // Developmental code should specify :dev -process.container = 'nfcore/eager:2.4.1' +process.container = 'nfcore/eager:dev' // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -414,7 +414,7 @@ manifest { description = 'A fully reproducible and state-of-the-art ancient DNA analysis pipeline' mainScript = 'main.nf' nextflowVersion = '>=20.07.1' - version = '2.4.1' + version = '2.4.2dev' } // Function to ensure that resource requirements don't go beyond From b27f7d1577724208c9d26f981c3f541246303290 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 3 Dec 2021 10:54:09 +0100 Subject: [PATCH 02/16] Add Megan to contributiors --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 284f9046d..58578aeb1 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ Those who have provided conceptual guidance, suggestions, bug reports etc. * [Katrin Nägele](https://github.com/KathrinNaegele) * [Luc Venturini](https://github.com/lucventurini) * [Marcel Keller](https://github.com/marcel-keller) +* [Megan Michel](https://github.com/meganemichel) * [Pierre Lindenbaum](https://github.com/lindenb) * [Pontus Skoglund](https://github.com/pontussk) * [Raphael Eisenhofer](https://github.com/EisenRa) From 7e93720960b813a444ca65cda1752f8bf7814aa0 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 3 Dec 2021 10:57:59 +0100 Subject: [PATCH 03/16] Typo katrhin's name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58578aeb1..1b4ff5b36 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Those who have provided conceptual guidance, suggestions, bug reports etc. * [Işın Altınkaya](https://github.com/isinaltinkaya) * [Johan Nylander](https://github.com/nylander) * [Katerine Eaton](https://github.com/ktmeaton) -* [Katrin Nägele](https://github.com/KathrinNaegele) +* [Kathrin Nägele](https://github.com/KathrinNaegele) * [Luc Venturini](https://github.com/lucventurini) * [Marcel Keller](https://github.com/marcel-keller) * [Megan Michel](https://github.com/meganemichel) From 005c150b7469d5b28bd6201eccc9bfb9f204d420 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 11 Jan 2022 20:00:24 +0100 Subject: [PATCH 04/16] Update main.nf --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 804b44609..d8ca5459c 100644 --- a/main.nf +++ b/main.nf @@ -1040,7 +1040,7 @@ if ( params.skip_collapse ){ // Inline barcode removal bypass when not running it if (params.run_post_ar_trimming) { ch_adapterremoval_for_skip_post_ar_trimming - .dump(tag: "inline_removal_bypass") + .mix( ch_post_ar_trimming_for_lanemerge ) .into { ch_inlinebarcoderemoval_for_fastqc_after_clipping; ch_inlinebarcoderemoval_for_lanemerge; } } else { ch_adapterremoval_for_skip_post_ar_trimming From b7fc2a4cc2592b7628fa45592b9e6b056afd08b9 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 11 Jan 2022 20:02:45 +0100 Subject: [PATCH 05/16] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 144b48f91..c467e7d84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### `Fixed` +- [#822](https://github.com/nf-core/eager/issues/822) Fixed post-adapterrmoeval trimmied files not being lane-merged and included in downstream analyses + ### `Dependencies` ### `Deprecated` From 3dbee54c76bcd8fc70278831fdcfc68b2108373b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 11 Jan 2022 20:36:12 +0100 Subject: [PATCH 06/16] Update main.nf --- main.nf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.nf b/main.nf index d8ca5459c..4683d8238 100644 --- a/main.nf +++ b/main.nf @@ -1039,8 +1039,7 @@ if ( params.skip_collapse ){ // Inline barcode removal bypass when not running it if (params.run_post_ar_trimming) { - ch_adapterremoval_for_skip_post_ar_trimming - .mix( ch_post_ar_trimming_for_lanemerge ) + ch_post_ar_trimming_for_lanemerge .into { ch_inlinebarcoderemoval_for_fastqc_after_clipping; ch_inlinebarcoderemoval_for_lanemerge; } } else { ch_adapterremoval_for_skip_post_ar_trimming From 0e6d09c1337be2368f84a64b20f1a99993afb76b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 11 Jan 2022 20:37:41 +0100 Subject: [PATCH 07/16] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c467e7d84..09fec65b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### `Fixed` -- [#822](https://github.com/nf-core/eager/issues/822) Fixed post-adapterrmoeval trimmied files not being lane-merged and included in downstream analyses +- [#822](https://github.com/nf-core/eager/issues/822) Fixed post-adapterremoval trimmed files not being lane-merged and included in downstream analyses ### `Dependencies` From 789c12e7e82d182a262c98b518e2f13fa34eda12 Mon Sep 17 00:00:00 2001 From: Thiseas Christos Lamnidis Date: Wed, 12 Jan 2022 17:10:48 +0100 Subject: [PATCH 08/16] bedtools: Add genome and sorted flags --- main.nf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main.nf b/main.nf index 4683d8238..3b489a8c4 100644 --- a/main.nf +++ b/main.nf @@ -2054,8 +2054,12 @@ process bedtools { script: """ - bedtools coverage -nonamecheck -a ${anno_file} -b $bam | pigz -p ${task.cpus - 1} > "${bam.baseName}".breadth.gz - bedtools coverage -nonamecheck -a ${anno_file} -b $bam -mean | pigz -p ${task.cpus - 1} > "${bam.baseName}".depth.gz + ## Create genome file from bam header + samtools view -H ${bam} | grep @SQ | sed 's/@SQ\tSN:\|LN://g' > genome.txt + + ## Run bedtools + bedtools coverage -nonamecheck -g genome.txt -sorted -a ${anno_file} -b ${bam} | pigz -p ${task.cpus - 1} > "${bam.baseName}".breadth.gz + bedtools coverage -nonamecheck -g genome.txt -sorted -a ${anno_file} -b ${bam} -mean | pigz -p ${task.cpus - 1} > "${bam.baseName}".depth.gz """ } From 180b6a76e8b6a40a535f66d8e1b092f0dd23e89c Mon Sep 17 00:00:00 2001 From: "Thiseas C. Lamnidis" Date: Wed, 12 Jan 2022 20:10:49 +0100 Subject: [PATCH 09/16] avoid slashes in sed, escape backslash --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 3b489a8c4..65813086f 100644 --- a/main.nf +++ b/main.nf @@ -2055,7 +2055,7 @@ process bedtools { script: """ ## Create genome file from bam header - samtools view -H ${bam} | grep @SQ | sed 's/@SQ\tSN:\|LN://g' > genome.txt + samtools view -H ${bam} | grep '@SQ' | sed 's#@SQ\tSN:\\|LN:##g' > genome.txt ## Run bedtools bedtools coverage -nonamecheck -g genome.txt -sorted -a ${anno_file} -b ${bam} | pigz -p ${task.cpus - 1} > "${bam.baseName}".breadth.gz From 3929b585d007fa49d2d767bbb17c13c20e3b0817 Mon Sep 17 00:00:00 2001 From: "Thiseas C. Lamnidis" Date: Wed, 12 Jan 2022 20:35:04 +0100 Subject: [PATCH 10/16] Fixed bedtools memory needs --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09fec65b7..de630e166 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### `Added` ### `Fixed` - +- [#824](https://github.com/nf-core/eager/issues/824) Fixes large memory footprint of bedtools coverage calculation. - [#822](https://github.com/nf-core/eager/issues/822) Fixed post-adapterremoval trimmed files not being lane-merged and included in downstream analyses ### `Dependencies` From b9622a37f816a1101a86bb1694616fff31b92ccb Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 12 Jan 2022 20:45:50 +0100 Subject: [PATCH 11/16] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de630e166..c95c21f1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### `Added` ### `Fixed` + - [#824](https://github.com/nf-core/eager/issues/824) Fixes large memory footprint of bedtools coverage calculation. - [#822](https://github.com/nf-core/eager/issues/822) Fixed post-adapterremoval trimmed files not being lane-merged and included in downstream analyses From 7dbea762ae18ac765bc62aa4802a6ebdb7b43eb4 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 14 Jan 2022 15:42:11 +0100 Subject: [PATCH 12/16] Version bump for patch release --- .github/workflows/ci.yml | 4 ++-- CHANGELOG.md | 2 +- Dockerfile | 4 ++-- environment.yml | 2 +- lib/NfcoreSchema.groovy | 2 +- nextflow.config | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21b7ae020..96b09dc60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,13 +37,13 @@ jobs: - name: Build new docker image if: env.MATCHED_FILES - run: docker build --no-cache . -t nfcore/eager:dev + run: docker build --no-cache . -t nfcore/eager:2.4.2 - name: Pull docker image if: ${{ !env.MATCHED_FILES }} run: | docker pull nfcore/eager:dev - docker tag nfcore/eager:dev nfcore/eager:dev + docker tag nfcore/eager:dev nfcore/eager:2.4.2 - name: Install Nextflow env: diff --git a/CHANGELOG.md b/CHANGELOG.md index c95c21f1a..086ebbfae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [dev] +## [2.4.2] - 2022-01-14 ### `Added` diff --git a/Dockerfile b/Dockerfile index cec616444..3cc4ec4ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY environment.yml / RUN conda env create --quiet -f /environment.yml && conda clean -a # Add conda installation dir to PATH (instead of doing 'conda activate') -ENV PATH /opt/conda/envs/nf-core-eager-2.4.2dev/bin:$PATH +ENV PATH /opt/conda/envs/nf-core-eager-2.4.2/bin:$PATH # Dump the details of the installed packages to a file for posterity -RUN conda env export --name nf-core-eager-2.4.2dev > nf-core-eager-2.4.2dev.yml \ No newline at end of file +RUN conda env export --name nf-core-eager-2.4.2 > nf-core-eager-2.4.2.yml \ No newline at end of file diff --git a/environment.yml b/environment.yml index b4b3365cb..0db40a045 100644 --- a/environment.yml +++ b/environment.yml @@ -1,6 +1,6 @@ # You can use this file to create a conda environment for this pipeline: # conda env create -f environment.yml -name: nf-core-eager-2.4.2dev +name: nf-core-eager-2.4.2 channels: - conda-forge - bioconda diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index c2390f966..52ee73043 100644 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -259,7 +259,7 @@ class NfcoreSchema { return new_params } - /* + /* * This method tries to read a JSON params file */ private static LinkedHashMap params_load(String json_schema) { diff --git a/nextflow.config b/nextflow.config index 1bddbb339..36c2a0355 100644 --- a/nextflow.config +++ b/nextflow.config @@ -284,7 +284,7 @@ params { // Container slug. Stable releases should specify release tag! // Developmental code should specify :dev -process.container = 'nfcore/eager:dev' +process.container = 'nfcore/eager:2.4.2' // Load base.config by default for all pipelines includeConfig 'conf/base.config' @@ -414,7 +414,7 @@ manifest { description = 'A fully reproducible and state-of-the-art ancient DNA analysis pipeline' mainScript = 'main.nf' nextflowVersion = '>=20.07.1' - version = '2.4.2dev' + version = '2.4.2' } // Function to ensure that resource requirements don't go beyond From bf5191cc412d7005c844c37cb24f3704061801d6 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 17 Jan 2022 21:05:56 +0100 Subject: [PATCH 13/16] Update lib/NfcoreSchema.groovy Co-authored-by: Harshil Patel --- lib/NfcoreSchema.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 52ee73043..c2390f966 100644 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -259,7 +259,7 @@ class NfcoreSchema { return new_params } - /* + /* * This method tries to read a JSON params file */ private static LinkedHashMap params_load(String json_schema) { From 2930f5f6566e44ed821e3bf3b799d61219e91774 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 17 Jan 2022 21:11:23 +0100 Subject: [PATCH 14/16] Update NfcoreSchema.groovy --- lib/NfcoreSchema.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index c2390f966..52ee73043 100644 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -259,7 +259,7 @@ class NfcoreSchema { return new_params } - /* + /* * This method tries to read a JSON params file */ private static LinkedHashMap params_load(String json_schema) { From 9ad31b5f4b7c6f7f4a8f1a03995c7a3b5fcf205c Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Mon, 24 Jan 2022 16:30:18 +0100 Subject: [PATCH 15/16] Fix software versioning --- CHANGELOG.md | 1 + bin/scrape_software_versions.py | 4 ++-- main.nf | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 086ebbfae..02de7d3f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - [#824](https://github.com/nf-core/eager/issues/824) Fixes large memory footprint of bedtools coverage calculation. - [#822](https://github.com/nf-core/eager/issues/822) Fixed post-adapterremoval trimmed files not being lane-merged and included in downstream analyses +- Fixed a couple of software version reporting commands ### `Dependencies` diff --git a/bin/scrape_software_versions.py b/bin/scrape_software_versions.py index ed7008381..41e4f2bcb 100755 --- a/bin/scrape_software_versions.py +++ b/bin/scrape_software_versions.py @@ -9,13 +9,13 @@ "FastQC": ["v_fastqc.txt", r"FastQC v(\S+)"], "MultiQC": ["v_multiqc.txt", r"multiqc, version (\S+)"], 'AdapterRemoval':['v_adapterremoval.txt', r"AdapterRemoval ver. (\S+)"], - 'Picard MarkDuplicates': ['v_markduplicates.txt', r"(\S+)"], + 'Picard MarkDuplicates': ['v_markduplicates.txt', r"Version:(\S+)"], 'Samtools': ['v_samtools.txt', r"samtools (\S+)"], 'Preseq': ['v_preseq.txt', r"Version: (\S+)"], 'BWA': ['v_bwa.txt', r"Version: (\S+)"], 'Bowtie2': ['v_bowtie2.txt', r"bowtie2-([0-9]+\.[0-9]+\.[0-9]+) -fdebug"], 'Qualimap': ['v_qualimap.txt', r"QualiMap v.(\S+)"], - 'GATK HaplotypeCaller': ['v_gatk.txt', r" v(\S+)"], + 'GATK HaplotypeCaller': ['v_gatk.txt', r"The Genome Analysis Toolkit \(GATK\) v(\S+)"], 'GATK UnifiedGenotyper': ['v_gatk3.txt', r"(\S+)"], 'bamUtil' : ['v_bamutil.txt', r"Version: (\S+);"], 'fastP': ['v_fastp.txt', r"([\d\.]+)"], diff --git a/main.nf b/main.nf index 65813086f..6f89cd0fa 100644 --- a/main.nf +++ b/main.nf @@ -3088,7 +3088,7 @@ process get_software_versions { ( exec 7>&1; picard MarkDuplicates --version 2>&1 >&7 | grep -v '/' >&2 ) 2> v_markduplicates.txt || true qualimap --version &> v_qualimap.txt 2>&1 || true preseq &> v_preseq.txt 2>&1 || true - gatk --version 2>&1 | head -n 1 > v_gatk.txt 2>&1 || true + gatk --version 2>&1 | grep '(GATK)' > v_gatk.txt 2>&1 || true gatk3 --version 2>&1 | head -n 1 > v_gatk3.txt 2>&1 || true freebayes --version &> v_freebayes.txt 2>&1 || true bedtools --version &> v_bedtools.txt 2>&1 || true From fa6f061ff65e02553dd84331da9fdfd5456d54ff Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 24 Jan 2022 17:59:34 +0100 Subject: [PATCH 16/16] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02de7d3f9..e479bc5aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## [2.4.2] - 2022-01-14 +## [2.4.2] - 2022-01-24 ### `Added`