chore: update github CI (#577)

.ci.condarc

@@ -0,0 +1,5 @@
+channels:
+  - conda-forge
+  - bioconda
+  - nodefaults
+channel_priority: strict

.github/actions/conda-cache/action.yml

@@ -0,0 +1,38 @@
+name: 'Install/Cache conda'
+description: 'Install or retrieve a conda environment from cache'
+inputs:
+  python-version:
+    required: true
+    type: string
+  environment-file:
+    required: true
+    type: string
+  cache-number:
+    required: false
+    type: number
+    default: 1
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: ci_test_conda
+        python-version: ${{ inputs.python-version }}
+
+    - name: Cache conda
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CONDA }}/envs
+        key: ${{ runner.os }}-conda-${{ inputs.cache-number }}-${{ hashFiles(inputs.environment-file) }}
+      id: cache
+
+    - name: Update conda environment
+      shell: bash -el {0}
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: conda env update -n ci_test_conda -f ${{ inputs.environment-file }}

.github/workflows/ci-e2e.yml

@@ -42,6 +42,9 @@ on:
       - 'snappy_wrappers/wrappers/somatic_variant_filtration/**'
       - 'snappy_wrappers/wrappers/vep/**'
 
+defaults:
+  run:
+    shell: bash -el {0}
 
 jobs:
 
@@ -51,30 +54,48 @@ jobs:
     - uses: actions/checkout@v4
       with:
         lfs: 'true'
+
+    - name: Setup conda
+      uses: ./.github/actions/conda-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+        environment-file: environment.yml
+
     - name: Test workflow (local FASTQs)
-      uses: snakemake/snakemake-github-action@v1
+      uses: snakemake/snakemake-github-action@cebcb23c87aa3a23efddff3f79b3a4bd143d4120
       with:
         directory: .tests/test-workflow
         snakefile: .tests/test-workflow/workflow/Snakefile
         args: "--configfile .tests/test-workflow/config/config.yaml --use-conda --show-failed-logs -j 2 --conda-cleanup-pkgs cache --dryrun"
         show-disk-usage-on-error: true
+        snakemake-version: 7.32.4
 
 
   Tests:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
     needs:
       - Dryrun_Tests
     steps:
-    - name: update apt
-      run: sudo apt-get update
     - uses: actions/checkout@v4
       with:
         lfs: 'true'
+
+    - name: Setup conda
+      uses: ./.github/actions/conda-cache
+      with:
+        python-version: ${{ matrix.python-version }}
+        environment-file: environment.yml
+
     - name: Test workflow (local FASTQs)
-      uses: snakemake/snakemake-github-action@v1
+      uses: snakemake/snakemake-github-action@cebcb23c87aa3a23efddff3f79b3a4bd143d4120
       with:
         directory: .tests/test-workflow
         snakefile: .tests/test-workflow/workflow/Snakefile
         args: "--configfile .tests/test-workflow/config/config.yaml --use-conda --show-failed-logs -j 2 --conda-cleanup-pkgs cache"
         show-disk-usage-on-error: true
+        snakemake-version: 7.32.4
 

.github/workflows/ci.yml

@@ -25,13 +25,18 @@ jobs:
 
   linting:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
-      - name: Install Python
-        uses: actions/setup-python@v2
+      - name: Setup conda
+        uses: ./.github/actions/conda-cache
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.python-version }}
+          environment-file: environment.yml
 
       - name: Install dependencies
         run: |
@@ -54,26 +59,6 @@ jobs:
             You can trigger all lints locally by running `make lint`
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-#  sphinx-docs:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - uses: actions/checkout@v4
-#
-#      - name: Install Python
-#        uses: actions/setup-python@v5
-#        with:
-#          python-version: "3.12"
-#
-#      - name: Install dependencies
-#        run: |
-#            pip install -e ".[docs]"
-#            pip freeze
-#
-#      - name: Build documentation
-#        run: |
-#          cd docs
-#          make html
-
   testing:
     runs-on: ubuntu-latest
     strategy:
@@ -87,15 +72,13 @@ jobs:
         with:
           lfs: true
           fetch-depth: 2
+
       - name: Setup conda
-        uses: conda-incubator/setup-miniconda@v3
+        uses: ./.github/actions/conda-cache
         with:
-          auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           environment-file: environment.yml
-          channels: conda-forge,bioconda,nodefaults
-          channel-priority: strict
-          activate-environment: test
+
       - run: which pip
       - name: Install some more dependencies via pip
         run: pip install '.[test]'  # 'pyproject.toml[test]'

.tests/test-workflow/workflow/envs/snappy.yaml

@@ -1,74 +1,72 @@
 channels:
   - conda-forge
   - bioconda
+  - nodefaults
+
 dependencies:
 
   # Fundamentals
-  - python >=3.12.0
-  - pip =24
-  - git-lfs =3.5
-  - gcc_linux-64 =13.2
-  - gxx_linux-64 =13.2
+  - python=3.12
+  - pip
+  - git-lfs ~=3.5.1
+
+  # for compiling packages from pip
+  - gcc_linux-64 ~=13.2.0
+  - gxx_linux-64 ~=13.2.0
 
   # basics some snappy wrappers rely on
-  - coreutils =9.5
-  - gawk =5.3
-  - bash =5.2
-  - gzip =1.13
+  - coreutils ~=9.5
+  - gawk ~=5.3.0
+  - bash ~=5.2.21
+  - gzip ~=1.13
+
+  # pydantic is used to validate configuration files
+  - pydantic =2.7
 
   # Snakemake is used for providing the actual wrapper calling functionality
-  - snakemake >=7,<8
+  - snakemake =7.32
 
   # Additional libraries used by snappy
-  - ruamel.yaml =0.18               # Nice, round-trip enabled YAML parsing
-  - fasteners =0.17                 # File-based locks
-  - termcolor =2.4                  # Helpful for CLIs
-  - matplotlib =3.8                 # Required for plotting
-  - jinja2 =3.1                     # Jinja 2 template rendering
+  - ruamel.yaml ==0.18.6    # Nice, round-trip enabled YAML parsing
+  - fasteners ==0.17.3      # File-based locks
+  - termcolor ==1.1.0       # Helpful for CLIs
+  - matplotlib ==3.8.4      # Required for plotting
+  - jinja2 ==3.1.4          # Jinja 2 template rendering
 
   # Bioinformatics-related libraries used by snappy
-  - htslib >=1.19
-  - bcftools >=1.19
-  - samtools >=1.19
-  # both vcfpy and pysam are only used in wrappers -- and tools, which should probably be wrappers in their own regard?
-  # current versions compatible with py>=3.11 aren't yet available from bioconda, so install via pip instead
-  # - vcfpy >=0.13.8                   # Library for working with VCF files
-  # - pysam                            # Support for vcfpy
-  # pytabix seems not to be used anywhere (and hasn't been updated in 9 years)
-  # nvm, it does see use in wrappers/vcfpy/add_bed/wrapper.py and wrappers/vcf_sv_filter/vcf_sv_filter.py
-  # - pytabix =0.1
+  - htslib ==1.20
+  - bcftools ==1.20
+  - samtools ==1.20
 
-  # Parsing of ISA-tab
-  - altamisa =0.2.9
+  # packages for testing
+  - pytest ~=8.2.2
+  - coverage ~=7.5.3
+  - pytest-cov ~=5.0.0
+  - pytest-mock ~=3.14.0
+  - pytest-subprocess ~=1.5.0
+  - pyfakefs ~=5.5.0
+  - pytest-sugar ~=0.9.6
+  - coveralls ~=4.0.1
+  - pytabix >=0.1.0,<1
+  # formatting, linting, dev
+  - ruff ~=0.4.8
+  - snakefmt ~=0.8.5
+  - pre-commit ~=3.7.1
+  # docs
+  - sphinx ~=7.3.7
+  - sphinx_rtd_theme ~=2.0.0
+  - sphinx-mdinclude ~=0.6.0
 
-  # miscellaneous
-  - attrs =23.2
+  - zlib ~=1.3.0
 
-  # Dependencies for testing
-  - pytest-sugar =1
-  - pytest =8.1
-  - pytest-cov =5.0
-  - coverage =7.4
-  - pytest-mock =3.14
-  - pytest-subprocess = 1.5
-  - pyfakefs =5.4
-  - snakefmt =0.10
-  - ruff =0.3
-  - coveralls =1.8
-
-  # Dependencies for documentation
-  - sphinx =7
-  - sphinx_rtd_theme =2
-  - sphinx-mdinclude =0.5
-
-  # CUBI libraries required by snappy (installed through pip)
   - pip:
-    - varfish-cli >=0.6.3
-    # We're trying to keep the PyPi package up to date, you might have to install
-    # from source, though.
-    # - biomedsheets >=0.11.7
-    # … which is what we do here:
-    - git+https://github.com/bihealth/biomedsheets.git@a355c396b6e6b2f52f96f73d58cd26c3f1f5e18c
-    - pysam >=0.22
-    - vcfpy >=0.13.8
-    - pytabix
+    # build varfish-cli from pypi
+    - varfish-cli ~=0.6.3
+    # specific compatible biomedsheets revision
+    - git+https://github.com/bihealth/biomedsheets.git@4e0a8484850c39d1511036c3fe29ec0b4f9271f8
+    # specific compatible altamisa revision
+    - git+https://github.com/bihealth/altamisa.git@817dc491ff819e4c80686082bf3e5f602f1ac14c
+
+    # build pysam and vcfpy via pip (to avoid python version + conda packaging issues)
+    - pysam ~=0.22
+    - vcfpy ~=0.13.8

environment.yml

@@ -59,6 +59,8 @@ dependencies:
   - sphinx_rtd_theme ~=2.0.0
   - sphinx-mdinclude ~=0.6.0
 
+  - zlib ~=1.3.0
+
   - pip:
     # build varfish-cli from pypi
     - varfish-cli ~=0.6.3