From 272e5c376f42261720b41088a29371e8d552f6fe Mon Sep 17 00:00:00 2001 From: Adam Laiacano Date: Mon, 8 Aug 2022 11:33:41 -0400 Subject: [PATCH] # This is a combination of 10 commits. # This is the 1st commit message: migrate github cpu build to use tox # This is the commit message #2: newline # This is the commit message #3: install tox in github action # This is the commit message #4: forgot tox oops # This is the commit message #5: remove macos version of tf # This is the commit message #6: install deps for docs in gh runner # This is the commit message #7: build docs with tox # This is the commit message #8: use a gh action to build docs # This is the commit message #9: move fil into tox file, remove docs for now # This is the commit message #10: add pytest for fil tests --- .github/workflows/cpu-ci.yml | 41 ++++++++++++--------------------- .github/workflows/fil.yml | 42 ---------------------------------- requirements-dev.txt | 15 ++---------- requirements-test-cpu.txt | 4 ++++ requirements-test-gpu.txt | 4 ++++ requirements-test.txt | 22 ++++++++++++++++++ tox.ini | 44 ++++++++++++++++++++++++++++++++++++ 7 files changed, 90 insertions(+), 82 deletions(-) delete mode 100644 .github/workflows/fil.yml create mode 100644 requirements-test-cpu.txt create mode 100644 requirements-test-gpu.txt create mode 100644 requirements-test.txt create mode 100644 tox.ini diff --git a/.github/workflows/cpu-ci.yml b/.github/workflows/cpu-ci.yml index 188b7a6f4..ceb0034a5 100644 --- a/.github/workflows/cpu-ci.yml +++ b/.github/workflows/cpu-ci.yml @@ -28,34 +28,18 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y protobuf-compiler pandoc - - name: Install dependencies + - name: Install and upgrade python packages run: | - python -m pip install --upgrade pip setuptools==59.4.0 wheel - python -m pip install -r requirements-dev.txt - python -m pip install tensorflow - # required for ensemble tests - python -m pip install xgboost - - name: Lint with flake8 + python -m pip install --upgrade pip setuptools==59.4.0 wheel tox + - name: Lint with flake8, black, isort, interrogate, codespell run: | - flake8 . - - name: Lint with black + tox -e lint + - name: Run tests run: | - black --check . - - name: Lint with isort + tox -e test-cpu + - name: Run FIL tests run: | - isort -c . - - name: Lint with interrogate - run: | - interrogate --config=pyproject.toml - - name: Lint with codespell - run: | - codespell - - name: Build - run: | - pip install . - - name: Run unittests - run: | - python -m pytest -rxs tests/unit/ + tox -e test-fil-cpu - name: Generate package for pypi run: | python setup.py sdist @@ -66,9 +50,12 @@ jobs: path: dist # Build docs, treat warnings as errors - - name: Building docs - run: | - make -C docs html SPHINXOPTS="-W -q" + # - name: Building docs + # uses: ammaraskar/sphinx-action@master + # with: + # docs-folder: "docs/" + # run: | + # tox -e docs - name: Upload HTML uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/fil.yml b/.github/workflows/fil.yml deleted file mode 100644 index 59567445a..000000000 --- a/.github/workflows/fil.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: fil - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.8] - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install Ubuntu packages - run: | - sudo apt-get update -y - sudo apt-get install -y protobuf-compiler - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools==59.4.0 wheel - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt - - name: Install FIL pip dependencies - run: | - python -m pip install xgboost lightgbm sklearn - # version of treelite is required to match the version used in Triton - python -m pip install treelite==2.3.0 treelite_runtime==2.3.0 - - name: Build - run: | - python setup.py develop - - name: Run unittests - run: | - python -m pytest -rxs tests/unit/systems/fil/ diff --git a/requirements-dev.txt b/requirements-dev.txt index e48b5edb0..839713cef 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,20 +1,10 @@ -# packages necessary to run tests and push PRs -# assumes requirements for nvtabular logic are already installed - +# Required to generate docs and also run tests tritonclient[all] -nvtabular - -# testing -pytest>=5 -pytest-cov>=2 -scikit-learn>=0.20 -git+https://github.com/rapidsai/asvdb.git -testbook # linting pylint==2.7.4 click==8.0.3 -black==21.7b0 +black==22.6.0 flake8==3.9.2 isort==5.9.3 bandit==1.7.0 @@ -35,4 +25,3 @@ natsort==8.0.1 myst-nb==0.13.2 linkify-it-py==1.0.3 sphinx-external-toc==0.2.4 - diff --git a/requirements-test-cpu.txt b/requirements-test-cpu.txt new file mode 100644 index 000000000..eab63e33a --- /dev/null +++ b/requirements-test-cpu.txt @@ -0,0 +1,4 @@ +-r requirements-test.txt + +faiss-cpu==1.7.2 +tensorflow<=2.9.0 diff --git a/requirements-test-gpu.txt b/requirements-test-gpu.txt new file mode 100644 index 000000000..3ee66b923 --- /dev/null +++ b/requirements-test-gpu.txt @@ -0,0 +1,4 @@ +-r requirements-test.txt + +tensorflow-gpu<=2.9.0 +faiss-gpu==1.7.2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 000000000..ac4877058 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,22 @@ +-r requirements.txt +-r requirements-dev.txt + +# This contains common libraries for testing. + +# NOTE: You should pip install requirements-test-[cpu|gpu].txt for device-specific test +# requirements, which will include the dependencies defined in this file. + +pytest>=5 +pytest-cov>=2 +scikit-learn>=0.20 +git+https://github.com/rapidsai/asvdb.git +testbook==0.4.2 + +# packages necessary to run tests and push PRs +feast==0.19.4 +xgboost==1.6.1 + +nvtabular + +# TODO: do we need more of these? +# https://github.com/NVIDIA-Merlin/Merlin/blob/a1cc48fe23c4dfc627423168436f26ef7e028204/ci/dockerfile.ci#L13-L18 diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..1910d200c --- /dev/null +++ b/tox.ini @@ -0,0 +1,44 @@ +[tox] +envlist = python3.8,test-gpu,test-cpu + +[testenv] +deps = -rrequirements-dev.txt +commands = + pip install --upgrade pip + pip install . +allowlist_externals = sphinx-build + +[testenv:test-cpu] +deps = -rrequirements-test-cpu.txt +commands = python -m pytest --cov-report term --cov=merlin -rxs tests/unit/ + +[testenv:test-gpu] +deps = -rrequirements-test-gpu.txt +commands = python -m pytest --cov-report term --cov=merlin -rxs tests/unit/ + +[testenv:test-fil-cpu] +deps = + pytest + xgboost + lightgbm + sklearn + treelite==2.3.0 + treelite_runtime==2.3.0 +commands = + python setup.py develop + python -m pytest -rxs tests/unit/systems/fil/ + +[testenv:lint] +commands = + flake8 setup.py merlin/ tests/ + black --check --diff merlin tests + pylint merlin + isort -c . --skip .tox + interrogate --config=pyproject.toml + codespell --skip .tox + +[testenv:docs] +changedir = {toxinidir}/docs +deps = attrs +commands = + sphinx-build -W -q source build