From 4ae78ab36d3affcfec87f9080f261243ab7a074b Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Tue, 1 Nov 2022 15:15:51 -0400 Subject: [PATCH] test(multiple): expand tests and developer docs * add DEVELOPER.md with basic install/testing info * add conftest.py with temp dir and misc fixtures * add tests for fixtures in conftest.py * add test for meson_build function * add PyGithub as test dep to setup.cfg * checkout modflow6 before testing in CI --- .github/workflows/ci.yml | 14 +++- DEVELOPER.md | 56 ++++++++++++++ modflow_devtools/build.py | 2 +- modflow_devtools/test/conftest.py | 101 +++++++++++++++++++++++++ modflow_devtools/test/test_build.py | 36 ++++++++- modflow_devtools/test/test_conftest.py | 89 ++++++++++++++++++++++ setup.cfg | 2 + 7 files changed, 296 insertions(+), 4 deletions(-) create mode 100644 DEVELOPER.md create mode 100644 modflow_devtools/test/conftest.py create mode 100644 modflow_devtools/test/test_conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68404bd..556f8a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,20 +62,32 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 + with: + path: modflow-devtools + + - name: Checkout modflow6 + uses: actions/checkout@v3 + with: + repository: MODFLOW-USGS/modflow6 + path: modflow6 - name: Setup Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} cache: 'pip' - cache-dependency-path: setup.cfg + cache-dependency-path: modflow-devtools/setup.cfg - name: Install Python packages + working-directory: modflow-devtools run: | pip3 install . pip3 install ".[test]" - name: Run tests + working-directory: modflow-devtools + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: pytest -v -n auto --durations 0 publish: diff --git a/DEVELOPER.md b/DEVELOPER.md new file mode 100644 index 0000000..222ecbe --- /dev/null +++ b/DEVELOPER.md @@ -0,0 +1,56 @@ +# Developing `modflow-devtools` + +This document provides guidance to set up a development environment and discusses conventions used in this project. + + + + +- [Developing `modflow-devtools`](#developing-modflow-devtools) + - [Installation](#installation) + - [Testing](#testing) + - [Environment variables](#environment-variables) + - [Running the tests](#running-the-tests) + + + +## Installation + +To get started, first fork and clone this repository. + +## Testing + +This project uses [`pytest`](https://docs.pytest.org/en/latest/) and several plugins. [`PyGithub`](https://github.com/PyGithub/PyGithub) is used to communicate with the GitHub API. + +### Environment variables + +#### `GITHUB_TOKEN` + +Tests require access to the GitHub API — in order to avoid rate limits, the tests attempt to authenticate with an access token. In C, this is the `GITHUB_TOKEN` provided by GitHub Actions. For local development a personal access token must be used. Setting the `GITHUB_TOKEN` variable manually will work, but the recommended approach is to use a `.env` file in your project root (the tests will automatically discover and use any environment variables configured here courtesy of [`pytest-dotenv`](https://github.com/quiqua/pytest-dotenv)). + +#### `MODFLOW6_PATH` + +By default, ths project's tests look for the `modflow6` repository side-by-side with `modflow-devtools` on the filesystem. The `MODFLOW6_PATH` variable is optional and can be used to configure a different location for the `modflow6` repo. + +### Running the tests + +To run the tests in parallel with verbose output, run from the project root: + +```shell +pytest -v -n auto +``` + +### Writing new tests + +Tests should follow a few conventions for ease of use and maintenance. + +#### Temporary directories + +If tests must write to disk, they should use `pytest`'s built-in `temp_dir` fixture or one of the scoped temporary directory fixtures defined in `conftest.py`. + +#### Using the GitHub API + +To access the GitHub API from a test case, just construct a `Github` object: + +```python +api = Github(environ.get("GITHUB_TOKEN")) +``` \ No newline at end of file diff --git a/modflow_devtools/build.py b/modflow_devtools/build.py index 97f1adf..9ed9b3c 100644 --- a/modflow_devtools/build.py +++ b/modflow_devtools/build.py @@ -34,7 +34,7 @@ def meson_build( print(f"Running: {cmd}") subprocess.run(cmd, shell=True, check=True) - cmd = "meson install -C builddir" + cmd = f"meson install -C {bld_path}" if not quiet: print(f"Running: {cmd}") subprocess.run(cmd, shell=True, check=True) diff --git a/modflow_devtools/test/conftest.py b/modflow_devtools/test/conftest.py new file mode 100644 index 0000000..29a438f --- /dev/null +++ b/modflow_devtools/test/conftest.py @@ -0,0 +1,101 @@ +from os import environ +from pathlib import Path + +import pytest +from github import Github + +proj_root = Path(__file__).parent.parent.parent.parent + + +# keepable temporary directory fixtures for various scopes + + +@pytest.fixture(scope="function") +def tmpdir(tmpdir_factory, request) -> Path: + node = ( + request.node.name.replace("/", "_") + .replace("\\", "_") + .replace(":", "_") + ) + temp = Path(tmpdir_factory.mktemp(node)) + yield Path(temp) + + keep = request.config.getoption("--keep") + if keep: + copytree(temp, Path(keep) / temp.name) + + keep_failed = request.config.getoption("--keep-failed") + if keep_failed and request.node.rep_call.failed: + copytree(temp, Path(keep_failed) / temp.name) + + +@pytest.fixture(scope="class") +def class_tmpdir(tmpdir_factory, request) -> Path: + assert ( + request.cls is not None + ), "Class-scoped temp dir fixture must be used on class" + temp = Path(tmpdir_factory.mktemp(request.cls.__name__)) + yield temp + + keep = request.config.getoption("--keep") + if keep: + copytree(temp, Path(keep) / temp.name) + + +@pytest.fixture(scope="module") +def module_tmpdir(tmpdir_factory, request) -> Path: + temp = Path(tmpdir_factory.mktemp(request.module.__name__)) + yield temp + + keep = request.config.getoption("--keep") + if keep: + copytree(temp, Path(keep) / temp.name) + + +@pytest.fixture(scope="session") +def session_tmpdir(tmpdir_factory, request) -> Path: + temp = Path(tmpdir_factory.mktemp(request.session.name)) + yield temp + + keep = request.config.getoption("--keep") + if keep: + copytree(temp, Path(keep) / temp.name) + + +# misc fixtures + + +@pytest.fixture +def gh_api() -> Github: + return Github(environ.get("GITHUB_TOKEN")) + + +@pytest.fixture +def modflow6_path() -> Path: + return Path(environ.get("MODFLOW6_PATH", proj_root / "modflow6")) + + +# pytest configuration hooks + + +def pytest_addoption(parser): + parser.addoption( + "-K", + "--keep", + action="store", + default=None, + help="Move the contents of temporary test directories to correspondingly named subdirectories at the given " + "location after tests complete. This option can be used to exclude test results from automatic cleanup, " + "e.g. for manual inspection. The provided path is created if it does not already exist. An error is " + "thrown if any matching files already exist.", + ) + + parser.addoption( + "--keep-failed", + action="store", + default=None, + help="Move the contents of temporary test directories to correspondingly named subdirectories at the given " + "location if the test case fails. This option automatically saves the outputs of failed tests in the " + "given location. The path is created if it doesn't already exist. An error is thrown if files with the " + "same names already exist in the given location.", + ) diff --git a/modflow_devtools/test/test_build.py b/modflow_devtools/test/test_build.py index 87a9b9a..7cb1194 100644 --- a/modflow_devtools/test/test_build.py +++ b/modflow_devtools/test/test_build.py @@ -1,2 +1,34 @@ -def test_meson_build(): - pass +import platform + +from modflow_devtools.build import meson_build + +system = platform.system() + + +def test_meson_build(modflow6_path, tmpdir): + bld_path = tmpdir / "builddir" + bin_path = tmpdir / "bin" + lib_path = bin_path + + meson_build(modflow6_path, bld_path, bin_path, bin_path, quiet=False) + + # check build directory was populated + assert (bld_path / "build.ninja").is_file() + assert (bld_path / "src").is_dir() + assert (bld_path / "meson-logs").is_dir() + + # check binaries and libraries were created + ext = ".exe" if system == "Windows" else "" + for exe in ["mf6", "mf5to6", "zbud6"]: + assert (bin_path / f"{exe}{ext}").is_file() + assert ( + bin_path + / ( + "libmf6" + + ( + ".so" + if system == "Linux" + else (".dylib" if system == "Darwin" else "") + ) + ) + ).is_file() diff --git a/modflow_devtools/test/test_conftest.py b/modflow_devtools/test/test_conftest.py new file mode 100644 index 0000000..3ed7ca2 --- /dev/null +++ b/modflow_devtools/test/test_conftest.py @@ -0,0 +1,89 @@ +import inspect +from os import environ +from pathlib import Path + +import pytest + +proj_root = Path(__file__).parent.parent.parent.parent + + +# test environment variables + + +def test_environment(): + assert environ.get("GITHUB_TOKEN") + assert Path(environ.get("MODFLOW6_PATH", proj_root / "modflow6")).is_dir() + + +# temporary directory fixtures + + +def test_tmpdirs(tmpdir, module_tmpdir): + # function-scoped temporary directory + assert isinstance(tmpdir, Path) + assert tmpdir.is_dir() + assert inspect.currentframe().f_code.co_name in tmpdir.stem + + # module-scoped temp dir (accessible to other tests in the script) + assert module_tmpdir.is_dir() + assert "test" in module_tmpdir.stem + + +def test_function_scoped_tmpdir(tmpdir): + assert isinstance(tmpdir, Path) + assert tmpdir.is_dir() + assert inspect.currentframe().f_code.co_name in tmpdir.stem + + +@pytest.mark.parametrize("name", ["noslash", "forward/slash", "back\\slash"]) +def test_function_scoped_tmpdir_slash_in_name(tmpdir, name): + assert isinstance(tmpdir, Path) + assert tmpdir.is_dir() + + # node name might have slashes if test function is parametrized + # (e.g., test_function_scoped_tmpdir_slash_in_name[a/slash]) + replaced1 = name.replace("/", "_").replace("\\", "_").replace(":", "_") + replaced2 = name.replace("/", "_").replace("\\", "__").replace(":", "_") + assert ( + f"{inspect.currentframe().f_code.co_name}[{replaced1}]" in tmpdir.stem + or f"{inspect.currentframe().f_code.co_name}[{replaced2}]" + in tmpdir.stem + ) + + +class TestClassScopedTmpdir: + filename = "hello.txt" + + @pytest.fixture(autouse=True) + def setup(self, class_tmpdir): + with open(class_tmpdir / self.filename, "w") as file: + file.write("hello, class-scoped tmpdir") + + def test_class_scoped_tmpdir(self, class_tmpdir): + assert isinstance(class_tmpdir, Path) + assert class_tmpdir.is_dir() + assert self.__class__.__name__ in class_tmpdir.stem + assert Path(class_tmpdir / self.filename).is_file() + + +def test_module_scoped_tmpdir(module_tmpdir): + assert isinstance(module_tmpdir, Path) + assert module_tmpdir.is_dir() + assert Path(inspect.getmodulename(__file__)).stem in module_tmpdir.name + + +def test_session_scoped_tmpdir(session_tmpdir): + assert isinstance(session_tmpdir, Path) + assert session_tmpdir.is_dir() + + +# test misc fixtures + + +def test_github_api(gh_api): + assert gh_api.get_user().login + + +def test_modflow6_path(modflow6_path): + assert modflow6_path.is_dir() + assert (modflow6_path / "version.txt").is_file() diff --git a/setup.cfg b/setup.cfg index 2777138..eedc767 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,8 +56,10 @@ test = %(lint)s coverage flaky + PyGithub pytest pytest-cov + pytest-dotenv pytest-xdist [flake8]