diff --git a/.ci.condarc b/.ci.condarc new file mode 100644 index 000000000..7bbe77253 --- /dev/null +++ b/.ci.condarc @@ -0,0 +1,5 @@ +channels: + - conda-forge + - bioconda + - nodefaults +channel_priority: strict diff --git a/.github/actions/conda-cache/action.yml b/.github/actions/conda-cache/action.yml new file mode 100644 index 000000000..a66de793c --- /dev/null +++ b/.github/actions/conda-cache/action.yml @@ -0,0 +1,38 @@ +name: 'Install/Cache conda' +description: 'Install or retrieve a conda environment from cache' +inputs: + python-version: + required: true + type: string + environment-file: + required: true + type: string + cache-number: + required: false + type: number + default: 1 + +defaults: + run: + shell: bash -el {0} + +runs: + using: "composite" + steps: + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ci_test_conda + python-version: ${{ inputs.python-version }} + + - name: Cache conda + uses: actions/cache@v3 + with: + path: ${{ env.CONDA }}/envs + key: ${{ runner.os }}-conda-${{ inputs.cache-number }}-${{ hashFiles(inputs.environment-file) }} + id: cache + + - name: Update conda environment + shell: bash -el {0} + if: steps.cache.outputs.cache-hit != 'true' + run: conda env update -n ci_test_conda -f ${{ inputs.environment-file }} diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index b9c11da2b..bc95e84df 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -42,6 +42,9 @@ on: - 'snappy_wrappers/wrappers/somatic_variant_filtration/**' - 'snappy_wrappers/wrappers/vep/**' +defaults: + run: + shell: bash -el {0} jobs: @@ -51,30 +54,48 @@ jobs: - uses: actions/checkout@v4 with: lfs: 'true' + + - name: Setup conda + uses: ./.github/actions/conda-cache + with: + python-version: ${{ matrix.python-version }} + environment-file: environment.yml + - name: Test workflow (local FASTQs) - uses: snakemake/snakemake-github-action@v1 + uses: snakemake/snakemake-github-action@cebcb23c87aa3a23efddff3f79b3a4bd143d4120 with: directory: .tests/test-workflow snakefile: .tests/test-workflow/workflow/Snakefile args: "--configfile .tests/test-workflow/config/config.yaml --use-conda --show-failed-logs -j 2 --conda-cleanup-pkgs cache --dryrun" show-disk-usage-on-error: true + snakemake-version: 7.32.4 Tests: runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.12" needs: - Dryrun_Tests steps: - - name: update apt - run: sudo apt-get update - uses: actions/checkout@v4 with: lfs: 'true' + + - name: Setup conda + uses: ./.github/actions/conda-cache + with: + python-version: ${{ matrix.python-version }} + environment-file: environment.yml + - name: Test workflow (local FASTQs) - uses: snakemake/snakemake-github-action@v1 + uses: snakemake/snakemake-github-action@cebcb23c87aa3a23efddff3f79b3a4bd143d4120 with: directory: .tests/test-workflow snakefile: .tests/test-workflow/workflow/Snakefile args: "--configfile .tests/test-workflow/config/config.yaml --use-conda --show-failed-logs -j 2 --conda-cleanup-pkgs cache" show-disk-usage-on-error: true + snakemake-version: 7.32.4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aada3f563..19a060214 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,13 +25,18 @@ jobs: linting: runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.12" steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - - name: Install Python - uses: actions/setup-python@v2 + - name: Setup conda + uses: ./.github/actions/conda-cache with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} + environment-file: environment.yml - name: Install dependencies run: | @@ -54,26 +59,6 @@ jobs: You can trigger all lints locally by running `make lint` GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# sphinx-docs: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# -# - name: Install Python -# uses: actions/setup-python@v5 -# with: -# python-version: "3.12" -# -# - name: Install dependencies -# run: | -# pip install -e ".[docs]" -# pip freeze -# -# - name: Build documentation -# run: | -# cd docs -# make html - testing: runs-on: ubuntu-latest strategy: @@ -87,15 +72,13 @@ jobs: with: lfs: true fetch-depth: 2 + - name: Setup conda - uses: conda-incubator/setup-miniconda@v3 + uses: ./.github/actions/conda-cache with: - auto-update-conda: true python-version: ${{ matrix.python-version }} environment-file: environment.yml - channels: conda-forge,bioconda,nodefaults - channel-priority: strict - activate-environment: test + - run: which pip - name: Install some more dependencies via pip run: pip install '.[test]' # 'pyproject.toml[test]' diff --git a/.tests/test-workflow/workflow/envs/snappy.yaml b/.tests/test-workflow/workflow/envs/snappy.yaml index 21ed2f92c..f10caf250 100644 --- a/.tests/test-workflow/workflow/envs/snappy.yaml +++ b/.tests/test-workflow/workflow/envs/snappy.yaml @@ -1,74 +1,72 @@ channels: - conda-forge - bioconda + - nodefaults + dependencies: # Fundamentals - - python >=3.12.0 - - pip =24 - - git-lfs =3.5 - - gcc_linux-64 =13.2 - - gxx_linux-64 =13.2 + - python=3.12 + - pip + - git-lfs ~=3.5.1 + + # for compiling packages from pip + - gcc_linux-64 ~=13.2.0 + - gxx_linux-64 ~=13.2.0 # basics some snappy wrappers rely on - - coreutils =9.5 - - gawk =5.3 - - bash =5.2 - - gzip =1.13 + - coreutils ~=9.5 + - gawk ~=5.3.0 + - bash ~=5.2.21 + - gzip ~=1.13 + + # pydantic is used to validate configuration files + - pydantic =2.7 # Snakemake is used for providing the actual wrapper calling functionality - - snakemake >=7,<8 + - snakemake =7.32 # Additional libraries used by snappy - - ruamel.yaml =0.18 # Nice, round-trip enabled YAML parsing - - fasteners =0.17 # File-based locks - - termcolor =2.4 # Helpful for CLIs - - matplotlib =3.8 # Required for plotting - - jinja2 =3.1 # Jinja 2 template rendering + - ruamel.yaml ==0.18.6 # Nice, round-trip enabled YAML parsing + - fasteners ==0.17.3 # File-based locks + - termcolor ==1.1.0 # Helpful for CLIs + - matplotlib ==3.8.4 # Required for plotting + - jinja2 ==3.1.4 # Jinja 2 template rendering # Bioinformatics-related libraries used by snappy - - htslib >=1.19 - - bcftools >=1.19 - - samtools >=1.19 - # both vcfpy and pysam are only used in wrappers -- and tools, which should probably be wrappers in their own regard? - # current versions compatible with py>=3.11 aren't yet available from bioconda, so install via pip instead - # - vcfpy >=0.13.8 # Library for working with VCF files - # - pysam # Support for vcfpy - # pytabix seems not to be used anywhere (and hasn't been updated in 9 years) - # nvm, it does see use in wrappers/vcfpy/add_bed/wrapper.py and wrappers/vcf_sv_filter/vcf_sv_filter.py - # - pytabix =0.1 + - htslib ==1.20 + - bcftools ==1.20 + - samtools ==1.20 - # Parsing of ISA-tab - - altamisa =0.2.9 + # packages for testing + - pytest ~=8.2.2 + - coverage ~=7.5.3 + - pytest-cov ~=5.0.0 + - pytest-mock ~=3.14.0 + - pytest-subprocess ~=1.5.0 + - pyfakefs ~=5.5.0 + - pytest-sugar ~=0.9.6 + - coveralls ~=4.0.1 + - pytabix >=0.1.0,<1 + # formatting, linting, dev + - ruff ~=0.4.8 + - snakefmt ~=0.8.5 + - pre-commit ~=3.7.1 + # docs + - sphinx ~=7.3.7 + - sphinx_rtd_theme ~=2.0.0 + - sphinx-mdinclude ~=0.6.0 - # miscellaneous - - attrs =23.2 + - zlib ~=1.3.0 - # Dependencies for testing - - pytest-sugar =1 - - pytest =8.1 - - pytest-cov =5.0 - - coverage =7.4 - - pytest-mock =3.14 - - pytest-subprocess = 1.5 - - pyfakefs =5.4 - - snakefmt =0.10 - - ruff =0.3 - - coveralls =1.8 - - # Dependencies for documentation - - sphinx =7 - - sphinx_rtd_theme =2 - - sphinx-mdinclude =0.5 - - # CUBI libraries required by snappy (installed through pip) - pip: - - varfish-cli >=0.6.3 - # We're trying to keep the PyPi package up to date, you might have to install - # from source, though. - # - biomedsheets >=0.11.7 - # … which is what we do here: - - git+https://github.com/bihealth/biomedsheets.git@a355c396b6e6b2f52f96f73d58cd26c3f1f5e18c - - pysam >=0.22 - - vcfpy >=0.13.8 - - pytabix + # build varfish-cli from pypi + - varfish-cli ~=0.6.3 + # specific compatible biomedsheets revision + - git+https://github.com/bihealth/biomedsheets.git@4e0a8484850c39d1511036c3fe29ec0b4f9271f8 + # specific compatible altamisa revision + - git+https://github.com/bihealth/altamisa.git@817dc491ff819e4c80686082bf3e5f602f1ac14c + + # build pysam and vcfpy via pip (to avoid python version + conda packaging issues) + - pysam ~=0.22 + - vcfpy ~=0.13.8 diff --git a/environment.yml b/environment.yml index 230480e29..c63c3dc14 100644 --- a/environment.yml +++ b/environment.yml @@ -59,6 +59,8 @@ dependencies: - sphinx_rtd_theme ~=2.0.0 - sphinx-mdinclude ~=0.6.0 + - zlib ~=1.3.0 + - pip: # build varfish-cli from pypi - varfish-cli ~=0.6.3