merge origin/main

bihealth · Dec 11, 2024 · d47884a · d47884a
2 parents b9834cb + 0d03919
commit d47884a
Show file tree

Hide file tree

Showing 15 changed files with 1,894 additions and 72 deletions.
diff --git a/.ci.condarc b/.ci.condarc
@@ -0,0 +1,5 @@
+channels:
+  - conda-forge
+  - bioconda
+  - nodefaults
+channel_priority: strict
diff --git a/.github/actions/conda-cache/action.yml b/.github/actions/conda-cache/action.yml
@@ -0,0 +1,38 @@
+name: 'Install/Cache conda'
+description: 'Install or retrieve a conda environment from cache'
+inputs:
+  python-version:
+    required: true
+    type: string
+  environment-file:
+    required: true
+    type: string
+  cache-number:
+    required: false
+    type: number
+    default: 1
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: ci_test_conda
+        python-version: ${{ inputs.python-version }}
+
+    - name: Cache conda
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CONDA }}/envs
+        key: ${{ runner.os }}-conda-${{ inputs.cache-number }}-${{ hashFiles(inputs.environment-file) }}
+      id: cache
+
+    - name: Update conda environment
+      shell: bash -el {0}
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: conda env update -n ci_test_conda -f ${{ inputs.environment-file }}
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml
@@ -42,6 +42,9 @@ on:
       - 'snappy_wrappers/wrappers/somatic_variant_filtration/**'
       - 'snappy_wrappers/wrappers/vep/**'
 
+defaults:
+  run:
+    shell: bash -el {0}
 
 jobs:
 
@@ -51,30 +54,48 @@ jobs:
     - uses: actions/checkout@v4
       with:
         lfs: 'true'
+
+    - name: Setup conda
+      uses: ./.github/actions/conda-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+        environment-file: environment.yml
+
     - name: Test workflow (local FASTQs)
-      uses: snakemake/snakemake-github-action@v1
+      uses: snakemake/snakemake-github-action@cebcb23c87aa3a23efddff3f79b3a4bd143d4120
       with:
         directory: .tests/test-workflow
         snakefile: .tests/test-workflow/workflow/Snakefile
         args: "--configfile .tests/test-workflow/config/config.yaml --use-conda --show-failed-logs -j 2 --conda-cleanup-pkgs cache --dryrun"
         show-disk-usage-on-error: true
+        snakemake-version: 7.32.4
 
 
   Tests:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
     needs:
       - Dryrun_Tests
     steps:
-    - name: update apt
-      run: sudo apt-get update
     - uses: actions/checkout@v4
       with:
         lfs: 'true'
+
+    - name: Setup conda
+      uses: ./.github/actions/conda-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+        environment-file: environment.yml
+
     - name: Test workflow (local FASTQs)
-      uses: snakemake/snakemake-github-action@v1
+      uses: snakemake/snakemake-github-action@cebcb23c87aa3a23efddff3f79b3a4bd143d4120
       with:
         directory: .tests/test-workflow
         snakefile: .tests/test-workflow/workflow/Snakefile
         args: "--configfile .tests/test-workflow/config/config.yaml --use-conda --show-failed-logs -j 2 --conda-cleanup-pkgs cache"
         show-disk-usage-on-error: true
+        snakemake-version: 7.32.4
 
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,13 +25,18 @@ jobs:
 
   linting:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
-      - name: Install Python
-        uses: actions/setup-python@v2
+      - name: Setup conda
+        uses: ./.github/actions/conda-cache
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
+          environment-file: environment.yml
 
       - name: Install dependencies
         run: |
@@ -54,26 +59,6 @@ jobs:
             You can trigger all lints locally by running `make lint`
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-#  sphinx-docs:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - uses: actions/checkout@v4
-#
-#      - name: Install Python
-#        uses: actions/setup-python@v5
-#        with:
-#          python-version: "3.12"
-#
-#      - name: Install dependencies
-#        run: |
-#            pip install -e ".[docs]"
-#            pip freeze
-#
-#      - name: Build documentation
-#        run: |
-#          cd docs
-#          make html
-
   testing:
     runs-on: ubuntu-latest
     strategy:
@@ -87,15 +72,13 @@ jobs:
         with:
           lfs: true
           fetch-depth: 2
+
       - name: Setup conda
-        uses: conda-incubator/setup-miniconda@v3
+        uses: ./.github/actions/conda-cache
         with:
-          auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           environment-file: environment.yml
-          channels: conda-forge,bioconda,nodefaults
-          channel-priority: strict
-          activate-environment: test
+
       - run: which pip
       - name: Install some more dependencies via pip
         run: pip install '.[test]'  # 'pyproject.toml[test]'

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,12 @@
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: v0.4.8
+    rev: v0.8.2
     hooks:
       - id: ruff
         args: [ --fix ]
       - id: ruff-format
   - repo: https://github.com/snakemake/snakefmt
-    rev: v0.8.5 # https://github.com/snakemake/snakefmt/releases
+    rev: v0.10.2 # https://github.com/snakemake/snakefmt/releases
     hooks:
       - id: snakefmt
diff --git a/.tests/test-workflow/workflow/envs/snappy.yaml b/.tests/test-workflow/workflow/envs/snappy.yaml
@@ -1,9 +1,7 @@
 channels:
   - conda-forge
   - bioconda
-  - defaults
-
-name: snappy_env
+  - nodefaults
 
 dependencies:
 
@@ -26,7 +24,7 @@ dependencies:
   - pydantic =2.7
 
   # Snakemake is used for providing the actual wrapper calling functionality
-  - snakemake >=8.14.0
+  - snakemake >=8.14.0,<9
 
   # Additional libraries used by snappy
   - ruamel.yaml ==0.18.6    # Nice, round-trip enabled YAML parsing
@@ -49,6 +47,7 @@ dependencies:
   - pyfakefs ~=5.5.0
   - pytest-sugar ~=0.9.6
   - coveralls ~=4.0.1
+  - pytabix >=0.1.0,<1
   # formatting, linting, dev
   - ruff ~=0.4.8
   - snakefmt ~=0.8.5
@@ -58,7 +57,7 @@ dependencies:
   - sphinx_rtd_theme ~=2.0.0
   - sphinx-mdinclude ~=0.6.0
 
-
+  - zlib ~=1.3.0
 
   - pip:
     # build varfish-cli from pypi

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.3.0](https://github.com/bihealth/snappy-pipeline/compare/v0.2.2...v0.3.0) (2024-12-09)
+
+
+### Features
+
+* enabled mbcs meta-tool in the ngs_mapping step model ([#545](https://github.com/bihealth/snappy-pipeline/issues/545)) ([625187b](https://github.com/bihealth/snappy-pipeline/commit/625187bd8c3088ece4c8228523dabb1d3f30893a))
+* Update GCNV environment to 0.9 kernel and GATK4.6.1 ([#557](https://github.com/bihealth/snappy-pipeline/issues/557)) ([7239fd0](https://github.com/bihealth/snappy-pipeline/commit/7239fd0b2f636f4cd0fa6bf1f420c38995ec4d07))
+
+
+### Bug Fixes
+
+* extra args check ([#551](https://github.com/bihealth/snappy-pipeline/issues/551)) ([7d6d649](https://github.com/bihealth/snappy-pipeline/commit/7d6d64925be88abc987662ce80793934ec9cea7f))
+* replace vep/run wrapper environment.yaml with symlink to vep environment.yaml ([#555](https://github.com/bihealth/snappy-pipeline/issues/555)) ([6873ffa](https://github.com/bihealth/snappy-pipeline/commit/6873ffaf68b3c95dc5ce2915cca6c3aae3fd0584))
+
 ## [0.2.2](https://github.com/bihealth/snappy-pipeline/compare/v0.2.1...v0.2.2) (2024-11-13)
 
 

diff --git a/environment.yml b/environment.yml
@@ -49,6 +49,7 @@ dependencies:
   - pyfakefs ~=5.5.0
   - pytest-sugar ~=0.9.6
   - coveralls ~=4.0.1
+  - pytabix >=0.1.0,<1
   # formatting, linting, dev
   - ruff ~=0.4.8
   - snakefmt ~=0.8.5
@@ -58,7 +59,7 @@ dependencies:
   - sphinx_rtd_theme ~=2.0.0
   - sphinx-mdinclude ~=0.6.0
 
-
+  - zlib ~=1.3.0
 
   - pip:
     # build varfish-cli from pypi

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools >= 61.0"]
+requires = ["setuptools >= 75.0"]
 build-backend = "setuptools.build_meta"
 
 
@@ -23,33 +23,31 @@ classifiers = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "setuptools ~=68.1.2",
     # Nice, round-trip enabled YAML parsing
-    "ruamel.yaml ~=0.18.6",
+    "ruamel.yaml >=0.18.6,<1",
     # File-based locks
-    "fasteners ~=0.17.3",
+    "fasteners >=0.17.3,<1",
     # We're trying to keep the PyPi package up to date, you might have to install
     # from source, though.
     "biomedsheets @ git+https://github.com/bihealth/biomedsheets.git@4e0a8484850c39d1511036c3fe29ec0b4f9271f8",
     # Helpful for CLIs
-    "termcolor ~=1.1.0",
+    "termcolor >=1.1.0,<3",
     # Snakemake is used for providing the actual wrapper calling functionality
-    "snakemake ~=8.14.0",
+    "snakemake >=8.14.0,<8",
     # Required for plotting
-    "matplotlib ~=3.8.4",
+    "matplotlib >=3.8.4",
     # Library for working with VCF files.
-    "vcfpy ~=0.13.8",
+    "vcfpy >=0.13.8,<1",
     # Support for vcfpy
-    "pysam ~=0.22.1",
-    "pytabix ~=0.1",
+    "pysam >=0.22.1,<1",
     # Jinja 2 template rendering
-    "jinja2 ~=3.1.4",
+    "jinja2 >=3.1.4,<4",
     # Parsing of ISA-tab.
     "altamisa @ git+https://github.com/bihealth/altamisa.git@817dc491ff819e4c80686082bf3e5f602f1ac14c",
     # REST API client for VarFish Server
-    "varfish-cli ~=0.6.3",
+    "varfish-cli >=0.6.3",
     # Validation for models, mainly used for configuration validation
-    "pydantic ~=2.7.0"
+    "pydantic >=2.9.0,<3"
 ]
 
 [project.urls]
@@ -60,28 +58,29 @@ Changelog = "https://github.com/bihealth/snappy-pipeline/blob/main/CHANGELOG.md"
 
 [project.optional-dependencies]
 test = [
-    "pytest ~=8.2.2",
-    "coverage ~=7.5.3",
-    "pytest-cov ~=5.0.0",
-    "pytest-mock ~=3.14.0",
-    "pytest-subprocess ~=1.5.0",
+    "pytest >=8.2.2,<9",
+    "coverage >=7.5.3,<8",
+    "pytest-cov >=5.0.0,<6",
+    "pytest-mock >=3.14.0,<4",
+    "pytest-subprocess >=1.5.0,<2",
     # Fake file system for testing
-    "pyfakefs ~=5.5.0",
-    "pytest-sugar ~=0.9.6",
+    "pyfakefs >=5.5.0,<6",
+    "pytest-sugar >=1.0.0,<2",
     # coveralls.io tooling
-    "coveralls ~=4.0.1",
+    "coveralls >=4.0.1,<5",
+    "pytabix >=0.1.0,<1"
 ]
 dev = [
     # ruff code linter + formatter
-    "ruff ~=0.4.8",
+    "ruff >=0.8.2,<1",
     # "snakefmt" code formatter and checker.
-    "snakefmt ~=0.8.5",
-    "pre-commit ~=3.7.1"
+    "snakefmt >=0.10.2,<1",
+    "pre-commit >=4.0.1,<5"
 ]
 docs = [
-    "sphinx ~=7.3.7",
-    "sphinx_rtd_theme ~=2.0.0",
-    "sphinx-mdinclude ~=0.6.0",
+    "sphinx >=7.3.7,<8",
+    "sphinx_rtd_theme >=2.0.0,<3",
+    "sphinx-mdinclude >=0.6.0,<1",
 ]
 all = [
     "snappy-pipeline[test,docs,dev]"

diff --git a/snappy_pipeline/version.py b/snappy_pipeline/version.py
@@ -1 +1 @@
-__version__ = "0.2.2"
+__version__ = "0.3.0"
diff --git a/tests/snappy_pipeline/workflows/test_workflows_variant_export_external.py b/tests/snappy_pipeline/workflows/test_workflows_variant_export_external.py
@@ -214,13 +214,15 @@ def test_workflow_check_config_invalid_search_pattern(
     # so pydantic tries to parse *2* dicts from the list and fails
     assert len(errors) == 2
 
+    import pydantic
+    pydantic_version = pydantic.version.version_short()
     expected_errors = [
         {
             "input": input_str,
             "loc": ("step_config", "variant_export_external", "search_patterns", i),
             "msg": "Input should be a valid dictionary",
             "type": "dict_type",
-            "url": "https://errors.pydantic.dev/2.7/v/dict_type",
+            "url": f"https://errors.pydantic.dev/{pydantic_version}/v/dict_type",
         }
         for i, input_str in enumerate(["vcf", "*/*.vcf.gz"])
     ]