feat: write a regression test suite that can be sbatch submitted #2978
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Tests | |
| on: | |
| - push | |
| - pull_request | |
| - workflow_call | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - run: sudo apt-get install sox libsox-dev ffmpeg | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| cache: "pip" | |
| - name: Install dependencies and package | |
| run: | | |
| CUDA_TAG=cpu pip install -r requirements.torch.txt --find-links https://download.pytorch.org/whl/torch_stable.html | |
| pip install -e .[dev] coverage | |
| - run: pip freeze | |
| - run: pip list | |
| - uses: tj-actions/changed-files@v45 | |
| id: file_changes | |
| with: | |
| # Only run pre-commit on EV files themselves, not on submodules | |
| # See https://github.com/EveryVoiceTTS/EveryVoice/issues/555 | |
| exclude_submodules: true | |
| - name: Custom replacement for pre-commit/action | |
| # pre-commit/action is not compatible with conda-incubator/setup-miniconda because it sets the shell wrong. | |
| run: python -m pre_commit run --show-diff-on-failure --color=always --files ${{ steps.file_changes.outputs.all_changed_files }} | |
| - uses: pre-commit-ci/[email protected] | |
| if: always() | |
| - name: Run tests | |
| run: | | |
| cd everyvoice && coverage run run_tests.py dev | |
| coverage xml | |
| - run: cd everyvoice && coverage report | |
| - name: Check for logs_and_checkpoints/ and preprocessed/ | |
| id: no-extra-directories | |
| run: | | |
| cd everyvoice && [[ $(find -type d -name logs_and_checkpoints -or -name preprocessed | grep --count --invert-match 'tests/data') -eq 0 ]] | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| fail_ci_if_error: false # optional (default = false) | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Make sure the CLI stays fast | |
| id: cli-load-time | |
| run: | | |
| PYTHONPROFILEIMPORTTIME=1 everyvoice -h 2> importtime.txt > /dev/null | |
| CLI_LOAD_TIME="$((/usr/bin/time --format=%E everyvoice -h > /dev/null) 2>&1)" | |
| echo "CLI load time: $CLI_LOAD_TIME" > import-message.txt | |
| PR_HEAD="${{ github.event.pull_request.head.sha }}" | |
| [[ $PR_HEAD ]] && echo "Pull Request HEAD: $PR_HEAD" >> import-message.txt | |
| echo "Imports that take more than 0.1 s:" >> import-message.txt | |
| grep -E 'cumulative|[0-9]{6} ' importtime.txt >> import-message.txt | |
| cat import-message.txt | |
| echo "Full import time log:" | |
| cat importtime.txt | |
| if [[ "$CLI_LOAD_TIME" > "0:01.00" ]]; then \ | |
| echo "ERROR: everyvoice --help is too slow."; \ | |
| echo "Please run 'PYTHONPROFILEIMPORTTIME=1 everyvoice -h 2> importtime.txt; tuna importtime.txt' and tuck away expensive imports so that the CLI doesn't load them until it uses them."; \ | |
| false; \ | |
| fi | |
| if grep -E -q "shared_types|pydantic" importtime.txt; then \ | |
| echo "ERROR: please be careful not to cause shared_types or pydantic to be imported when the CLI just loads. They are expensive imports."; \ | |
| false; \ | |
| fi | |
| - name: Report help speed in PR | |
| if: github.event_name == 'pull_request' | |
| uses: mshick/add-pr-comment@v2 | |
| with: | |
| preformatted: true | |
| message-path: import-message.txt | |
| - name: Check licenses | |
| run: | | |
| pip install licensecheck --no-warn-conflicts | |
| # Make sure we don't have or introduce dependencies with incompatible licenses. | |
| # Ignore packages where auto analysis does not work but manual analysis says OK. | |
| # Ignore nvidia-*-cu12 packages used by the GPU back-end: OTHER/PROPRIETARY LICENSE | |
| licensecheck --zero \ | |
| --ignore-packages text-unidecode pympi-ling pyworld pysdtw audioread \ | |
| --ignore-licenses OTHER/PROPRIETARY \ | |
| || echo "Package(s) listed with an X above is/are potentially a problem." | |
| # Only nvidia's and aiohappyeyeballs's proprietary licenses are OK. | |
| echo "" | |
| ! licensecheck --format csv 2> /dev/null | grep -v -E 'nvidia-|aiohappyeyeballs' | grep PROPRIETARY \ | |
| || echo "The package(s) listed just above this line is/are potentially a problem." |