Standardize directory structure

moiexpositoalonsolab · Jul 11, 2024 · f66d941 · f66d941
1 parent 2fee981
commit f66d941
Show file tree

Hide file tree

Showing 99 changed files with 24 additions and 13 deletions.
diff --git a/.snakemake-workflow-catalog.yml b/.snakemake-workflow-catalog.yml
@@ -0,0 +1,4 @@
+usage:
+  software-stack-deployment:
+    conda: true
+  report: true
diff --git a/LICENSE → LICENSE.txt b/LICENSE → LICENSE.txt
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 
 ![grenepipe logo](/doc/logo/grenepipe.png?raw=true)
 
-Snakemake pipeline for variant calling from raw sample sequences, with lots of bells and whistles.
+Snakemake workflow for variant calling from raw sample sequences, with lots of bells and whistles.
 
 **Advantages**:
 
@@ -23,8 +23,8 @@ Getting Started
 See [**--&gt; the Wiki pages &lt;--**](https://github.com/lczech/grenepipe/wiki) for setup and documentation.
 
 For **questions, bug reports, and feature requests**, please
-[open an issue](https://github.com/lczech/grenepipe/issues). 
-Please do not send emails with questions or requests, as others might be having them as well, 
+[open an issue](https://github.com/lczech/grenepipe/issues).
+Please do not send emails with questions or requests, as others might be having them as well,
 and so it is better to discuss them where they can be found.
 
 Pipeline Overview

diff --git a/config/README.md b/config/README.md
@@ -0,0 +1,7 @@
+# Configuring grenepipe
+
+The `config.yaml` contains all settings concerning the data, tools, and their parameters
+that are needed for analyzing your data. The file itself contains comments explaining
+each setting - see there for details.
+
+Furthermore, for the general usage of grenepipe, see our [wiki](https://github.com/moiexpositoalonsolab/grenepipe/wiki). See there to get started with grenepipe.
diff --git a/config.yaml → config/config.yaml b/config.yaml → config/config.yaml
diff --git a/test/run.sh b/test/run.sh
@@ -55,7 +55,7 @@ cat ./test/samples.tsv | egrep -v "^S3" > ./test/samples-pe.tsv
 make_config() {
     local TARGET=$1
 
-    cp ./config.yaml ${TARGET}
+    cp ./config/config.yaml ${TARGET}
     # cat ./config.yaml | sed "s?#BASEPATH#?${BASEPATH}?g" > ${TARGET}
     sed -i.bak -e "s?/path/to/data/samples.tsv?${BASEPATH}/test/samples.tsv?g" ${TARGET}
     sed -i.bak -e "s?/path/to/data/genome.fa?${BASEPATH}/test/reference/TAIR10_chr_all.fa?g" ${TARGET}

diff --git a/Snakefile → workflow/Snakefile b/Snakefile → workflow/Snakefile
@@ -4,7 +4,7 @@
 
 # We first need to load the common functionality, which gives us access to the config file,
 # generates the sample list, and prepares some other things for us that are needed below.
-include: "rules/common.smk"
+include: "rules/init.smk"
 
 # =================================================================================================
 #     Default "All" Target Rule

diff --git a/envs/adapterremoval.yaml → workflow/envs/adapterremoval.yaml b/envs/adapterremoval.yaml → workflow/envs/adapterremoval.yaml
diff --git a/envs/bamutil.yaml → workflow/envs/bamutil.yaml b/envs/bamutil.yaml → workflow/envs/bamutil.yaml
diff --git a/envs/bcftools.yaml → workflow/envs/bcftools.yaml b/envs/bcftools.yaml → workflow/envs/bcftools.yaml
diff --git a/envs/bedops.yaml → workflow/envs/bedops.yaml b/envs/bedops.yaml → workflow/envs/bedops.yaml
diff --git a/envs/bowtie2.yaml → workflow/envs/bowtie2.yaml b/envs/bowtie2.yaml → workflow/envs/bowtie2.yaml
diff --git a/envs/bwa-mem2.yaml → workflow/envs/bwa-mem2.yaml b/envs/bwa-mem2.yaml → workflow/envs/bwa-mem2.yaml
diff --git a/envs/bwa.yaml → workflow/envs/bwa.yaml b/envs/bwa.yaml → workflow/envs/bwa.yaml
diff --git a/envs/cutadapt.yaml → workflow/envs/cutadapt.yaml b/envs/cutadapt.yaml → workflow/envs/cutadapt.yaml
diff --git a/envs/damageprofiler.yaml → workflow/envs/damageprofiler.yaml b/envs/damageprofiler.yaml → workflow/envs/damageprofiler.yaml
diff --git a/envs/dedup.yaml → workflow/envs/dedup.yaml b/envs/dedup.yaml → workflow/envs/dedup.yaml
diff --git a/envs/fastqc.yaml → workflow/envs/fastqc.yaml b/envs/fastqc.yaml → workflow/envs/fastqc.yaml
diff --git a/envs/freebayes.yaml → workflow/envs/freebayes.yaml b/envs/freebayes.yaml → workflow/envs/freebayes.yaml
diff --git a/envs/frequency-table.yaml → workflow/envs/frequency-table.yaml b/envs/frequency-table.yaml → workflow/envs/frequency-table.yaml
diff --git a/envs/gatk.yaml → workflow/envs/gatk.yaml b/envs/gatk.yaml → workflow/envs/gatk.yaml
diff --git a/envs/grenepipe.yaml → workflow/envs/grenepipe.yaml b/envs/grenepipe.yaml → workflow/envs/grenepipe.yaml
diff --git a/envs/hafpipe.yaml → workflow/envs/hafpipe.yaml b/envs/hafpipe.yaml → workflow/envs/hafpipe.yaml
diff --git a/envs/mapdamage-linux.yaml → workflow/envs/mapdamage-linux.yaml b/envs/mapdamage-linux.yaml → workflow/envs/mapdamage-linux.yaml
diff --git a/envs/mapdamage-macos.yaml → workflow/envs/mapdamage-macos.yaml b/envs/mapdamage-macos.yaml → workflow/envs/mapdamage-macos.yaml
diff --git a/envs/multiqc.yaml → workflow/envs/multiqc.yaml b/envs/multiqc.yaml → workflow/envs/multiqc.yaml
diff --git a/envs/picard.yaml → workflow/envs/picard.yaml b/envs/picard.yaml → workflow/envs/picard.yaml
diff --git a/envs/qualimap.yaml → workflow/envs/qualimap.yaml b/envs/qualimap.yaml → workflow/envs/qualimap.yaml
diff --git a/envs/rbt.yaml → workflow/envs/rbt.yaml b/envs/rbt.yaml → workflow/envs/rbt.yaml
diff --git a/envs/samtools.yaml → workflow/envs/samtools.yaml b/envs/samtools.yaml → workflow/envs/samtools.yaml
diff --git a/envs/seqkit.yaml → workflow/envs/seqkit.yaml b/envs/seqkit.yaml → workflow/envs/seqkit.yaml
diff --git a/envs/seqprep-linux.yaml → workflow/envs/seqprep-linux.yaml b/envs/seqprep-linux.yaml → workflow/envs/seqprep-linux.yaml
diff --git a/envs/skewer.yaml → workflow/envs/skewer.yaml b/envs/skewer.yaml → workflow/envs/skewer.yaml
diff --git a/envs/snpeff.yaml → workflow/envs/snpeff.yaml b/envs/snpeff.yaml → workflow/envs/snpeff.yaml
diff --git a/envs/stats.yaml → workflow/envs/stats.yaml b/envs/stats.yaml → workflow/envs/stats.yaml
diff --git a/envs/tabix.yaml → workflow/envs/tabix.yaml b/envs/tabix.yaml → workflow/envs/tabix.yaml
diff --git a/envs/trimmomatic.yaml → workflow/envs/trimmomatic.yaml b/envs/trimmomatic.yaml → workflow/envs/trimmomatic.yaml
diff --git a/envs/vcflib.yaml → workflow/envs/vcflib.yaml b/envs/vcflib.yaml → workflow/envs/vcflib.yaml
diff --git a/envs/vep.yaml → workflow/envs/vep.yaml b/envs/vep.yaml → workflow/envs/vep.yaml
diff --git a/reports/calls.rst → workflow/report/calls.rst b/reports/calls.rst → workflow/report/calls.rst
diff --git a/reports/depths.rst → workflow/report/depths.rst b/reports/depths.rst → workflow/report/depths.rst
diff --git a/reports/freqs.rst → workflow/report/freqs.rst b/reports/freqs.rst → workflow/report/freqs.rst
diff --git a/reports/multiqc.rst → workflow/report/multiqc.rst b/reports/multiqc.rst → workflow/report/multiqc.rst
diff --git a/reports/stats.rst → workflow/report/stats.rst b/reports/stats.rst → workflow/report/stats.rst
diff --git a/reports/vcf.rst → workflow/report/vcf.rst b/reports/vcf.rst → workflow/report/vcf.rst
diff --git a/reports/workflow.rst → workflow/report/workflow.rst b/reports/workflow.rst → workflow/report/workflow.rst
diff --git a/rules/annotation.smk → workflow/rules/annotation.smk b/rules/annotation.smk → workflow/rules/annotation.smk
@@ -70,7 +70,7 @@ rule snpeff:
         # annotated calls (vcf, bcf, or vcf.gz)
         calls=report(
             "annotated/snpeff.vcf.gz",
-            caption="../reports/vcf.rst",
+            caption="../report/vcf.rst",
             category="Calls"
         ),
 

diff --git a/rules/calling-bcftools-combined.smk → workflow/rules/calling-bcftools-combined.smk b/rules/calling-bcftools-combined.smk → workflow/rules/calling-bcftools-combined.smk
diff --git a/rules/calling-bcftools-individual.smk → ...low/rules/calling-bcftools-individual.smk b/rules/calling-bcftools-individual.smk → ...low/rules/calling-bcftools-individual.smk
diff --git a/rules/calling-bcftools.smk → workflow/rules/calling-bcftools.smk b/rules/calling-bcftools.smk → workflow/rules/calling-bcftools.smk
diff --git a/rules/calling-freebayes.smk → workflow/rules/calling-freebayes.smk b/rules/calling-freebayes.smk → workflow/rules/calling-freebayes.smk
diff --git a/rules/calling-haplotypecaller.smk → workflow/rules/calling-haplotypecaller.smk b/rules/calling-haplotypecaller.smk → workflow/rules/calling-haplotypecaller.smk
diff --git a/rules/calling.smk → workflow/rules/calling.smk b/rules/calling.smk → workflow/rules/calling.smk
diff --git a/rules/damage.smk → workflow/rules/damage.smk b/rules/damage.smk → workflow/rules/damage.smk
diff --git a/rules/duplicates-dedup.smk → workflow/rules/duplicates-dedup.smk b/rules/duplicates-dedup.smk → workflow/rules/duplicates-dedup.smk
diff --git a/rules/duplicates-picard.smk → workflow/rules/duplicates-picard.smk b/rules/duplicates-picard.smk → workflow/rules/duplicates-picard.smk
diff --git a/rules/filtering-bcftools-filter.smk → workflow/rules/filtering-bcftools-filter.smk b/rules/filtering-bcftools-filter.smk → workflow/rules/filtering-bcftools-filter.smk
diff --git a/rules/filtering-gatk-variantfiltration.smk → ...ules/filtering-gatk-variantfiltration.smk b/rules/filtering-gatk-variantfiltration.smk → ...ules/filtering-gatk-variantfiltration.smk
diff --git a/rules/filtering-gatk-vqsr.smk → workflow/rules/filtering-gatk-vqsr.smk b/rules/filtering-gatk-vqsr.smk → workflow/rules/filtering-gatk-vqsr.smk
diff --git a/rules/filtering.smk → workflow/rules/filtering.smk b/rules/filtering.smk → workflow/rules/filtering.smk
diff --git a/rules/frequency.smk → workflow/rules/frequency.smk b/rules/frequency.smk → workflow/rules/frequency.smk
@@ -15,7 +15,7 @@ def get_packages_dir():
     # https://snakemake.readthedocs.io/en/stable/project_info/faq.html#how-does-snakemake-interpret-relative-paths
     # However, neither of them is working... So we use a different way of getting the path
     # to our Snakemake base directory: https://stackoverflow.com/a/73202976/4184258
-    return os.path.join( snakemake.workflow.basedir, "packages" )
+    return os.path.join( snakemake.workflow.basedir, "../packages" )
 
 def get_hafpipe_bins():
     harp_bin    = os.path.join( get_packages_dir(), "harp/bin/harp" )

diff --git a/rules/common.smk → workflow/rules/init.smk b/rules/common.smk → workflow/rules/init.smk
diff --git a/rules/mapping-bowtie2.smk → workflow/rules/mapping-bowtie2.smk b/rules/mapping-bowtie2.smk → workflow/rules/mapping-bowtie2.smk
diff --git a/rules/mapping-bwa-aln.smk → workflow/rules/mapping-bwa-aln.smk b/rules/mapping-bwa-aln.smk → workflow/rules/mapping-bwa-aln.smk
diff --git a/rules/mapping-bwa-mem.smk → workflow/rules/mapping-bwa-mem.smk b/rules/mapping-bwa-mem.smk → workflow/rules/mapping-bwa-mem.smk
diff --git a/rules/mapping-bwa-mem2.smk → workflow/rules/mapping-bwa-mem2.smk b/rules/mapping-bwa-mem2.smk → workflow/rules/mapping-bwa-mem2.smk
diff --git a/rules/mapping-recalibrate.smk → workflow/rules/mapping-recalibrate.smk b/rules/mapping-recalibrate.smk → workflow/rules/mapping-recalibrate.smk
diff --git a/rules/mapping.smk → workflow/rules/mapping.smk b/rules/mapping.smk → workflow/rules/mapping.smk
diff --git a/rules/pileup.smk → workflow/rules/pileup.smk b/rules/pileup.smk → workflow/rules/pileup.smk
diff --git a/rules/prep.smk → workflow/rules/prep.smk b/rules/prep.smk → workflow/rules/prep.smk
@@ -182,7 +182,7 @@ rule reference_seqkit:
 # =================================================================================================
 
 # We need the variants entry in the config to be either an empty list or a file path, which we
-# already ensure in common.smk. This is because snakemake does not accept empty strings as input
+# already ensure in init.smk. This is because snakemake does not accept empty strings as input
 # files - it has to be either a file path or an empty list. So here we need to do a bit of trickery
 # to allow for the case that no known variants file is given in the config:
 # We define a local variable that is always a string. If it is empty, that does not seem to matter

diff --git a/rules/qc-bam.smk → workflow/rules/qc-bam.smk b/rules/qc-bam.smk → workflow/rules/qc-bam.smk
diff --git a/rules/qc-fastq.smk → workflow/rules/qc-fastq.smk b/rules/qc-fastq.smk → workflow/rules/qc-fastq.smk
diff --git a/rules/qc-vcf.smk → workflow/rules/qc-vcf.smk b/rules/qc-vcf.smk → workflow/rules/qc-vcf.smk
diff --git a/rules/qc.smk → workflow/rules/qc.smk b/rules/qc.smk → workflow/rules/qc.smk
@@ -43,7 +43,7 @@ rule multiqc:
         "damageprofiler/damageprofiler.done" if config["settings"]["damageprofiler"] else []
 
     output:
-        report("qc/multiqc.html", caption="../reports/multiqc.rst", category="Quality control"),
+        report("qc/multiqc.html", caption="../report/multiqc.rst", category="Quality control"),
         "qc/multiqc.zip",
     params:
         config["params"]["multiqc"]["extra"],

diff --git a/rules/stats.smk → workflow/rules/stats.smk b/rules/stats.smk → workflow/rules/stats.smk
@@ -6,7 +6,7 @@ rule vcf_to_tsv:
     input:
         "annotated/snpeff.vcf.gz"
     output:
-        report("tables/calls.tsv.gz", caption="../reports/calls.rst", category="Calls")
+        report("tables/calls.tsv.gz", caption="../report/calls.rst", category="Calls")
     log:
         "logs/vcf_to_tsv.log"
     conda:
@@ -30,8 +30,8 @@ rule plot_stats:
     input:
         "tables/calls.tsv.gz"
     output:
-        depths=report("plots/depths.svg", caption="../reports/depths.rst", category="Plots"),
-        freqs=report("plots/allele-freqs.svg", caption="../reports/freqs.rst", category="Plots")
+        depths=report("plots/depths.svg", caption="../report/depths.rst", category="Plots"),
+        freqs=report("plots/allele-freqs.svg", caption="../report/freqs.rst", category="Plots")
     log:
         "logs/plot-depths.log"
     conda:

diff --git a/rules/trimming-adapterremoval.smk → workflow/rules/trimming-adapterremoval.smk b/rules/trimming-adapterremoval.smk → workflow/rules/trimming-adapterremoval.smk
diff --git a/rules/trimming-cutadapt.smk → workflow/rules/trimming-cutadapt.smk b/rules/trimming-cutadapt.smk → workflow/rules/trimming-cutadapt.smk
diff --git a/rules/trimming-fastp.smk → workflow/rules/trimming-fastp.smk b/rules/trimming-fastp.smk → workflow/rules/trimming-fastp.smk
diff --git a/rules/trimming-seqprep.smk → workflow/rules/trimming-seqprep.smk b/rules/trimming-seqprep.smk → workflow/rules/trimming-seqprep.smk
diff --git a/rules/trimming-skewer.smk → workflow/rules/trimming-skewer.smk b/rules/trimming-skewer.smk → workflow/rules/trimming-skewer.smk
diff --git a/rules/trimming-trimmomatic.smk → workflow/rules/trimming-trimmomatic.smk b/rules/trimming-trimmomatic.smk → workflow/rules/trimming-trimmomatic.smk
diff --git a/rules/trimming.smk → workflow/rules/trimming.smk b/rules/trimming.smk → workflow/rules/trimming.smk
diff --git a/schemas/config.schema.yaml → workflow/schemas/config.schema.yaml b/schemas/config.schema.yaml → workflow/schemas/config.schema.yaml
diff --git a/schemas/samples.schema.yaml → workflow/schemas/samples.schema.yaml b/schemas/samples.schema.yaml → workflow/schemas/samples.schema.yaml
diff --git a/schemas/yamllint.yaml → workflow/schemas/yamllint.yaml b/schemas/yamllint.yaml → workflow/schemas/yamllint.yaml
diff --git a/scripts/bwa-mem2-mem.py → workflow/scripts/bwa-mem2-mem.py b/scripts/bwa-mem2-mem.py → workflow/scripts/bwa-mem2-mem.py
diff --git a/scripts/bwa-samxe.py → workflow/scripts/bwa-samxe.py b/scripts/bwa-samxe.py → workflow/scripts/bwa-samxe.py
diff --git a/scripts/fastqc.py → workflow/scripts/fastqc.py b/scripts/fastqc.py → workflow/scripts/fastqc.py
diff --git a/scripts/freebayes.py → workflow/scripts/freebayes.py b/scripts/freebayes.py → workflow/scripts/freebayes.py
diff --git a/scripts/frequency-table.py → workflow/scripts/frequency-table.py b/scripts/frequency-table.py → workflow/scripts/frequency-table.py
diff --git a/scripts/hafpipe-concat.py → workflow/scripts/hafpipe-concat.py b/scripts/hafpipe-concat.py → workflow/scripts/hafpipe-concat.py
diff --git a/scripts/hafpipe-merge.py → workflow/scripts/hafpipe-merge.py b/scripts/hafpipe-merge.py → workflow/scripts/hafpipe-merge.py
diff --git a/scripts/hafpipe.py → workflow/scripts/hafpipe.py b/scripts/hafpipe.py → workflow/scripts/hafpipe.py
@@ -28,7 +28,7 @@
 # For all these steps, we get the directory where this python script file is in,
 # and then work our way up from there through our grenepipe directory.
 script_path = os.path.dirname(os.path.realpath(__file__))
-packages_path = os.path.join( script_path, "../packages" )
+packages_path = os.path.join( script_path, "../../packages" )
 
 harp_path   = os.path.join( packages_path, "harp/bin/" )
 harp_bin    = os.path.join( packages_path, "harp/bin/harp" )

diff --git a/scripts/multiqc.py → workflow/scripts/multiqc.py b/scripts/multiqc.py → workflow/scripts/multiqc.py
diff --git a/scripts/picard-collectmultiplemetrics.py → .../scripts/picard-collectmultiplemetrics.py b/scripts/picard-collectmultiplemetrics.py → .../scripts/picard-collectmultiplemetrics.py
diff --git a/scripts/plot-depths.py → workflow/scripts/plot-depths.py b/scripts/plot-depths.py → workflow/scripts/plot-depths.py
diff --git a/scripts/sample-sizes.py → workflow/scripts/sample-sizes.py b/scripts/sample-sizes.py → workflow/scripts/sample-sizes.py
diff --git a/scripts/vep-cache.py → workflow/scripts/vep-cache.py b/scripts/vep-cache.py → workflow/scripts/vep-cache.py
diff --git a/scripts/vep-plugins.py → workflow/scripts/vep-plugins.py b/scripts/vep-plugins.py → workflow/scripts/vep-plugins.py