Merge pull request #61 from MODFLOW-USGS/v0.1.4

* ci(release): update to development version 0.1.4 * docs(fixtures): add docs for model filtering by name/package (#52) * docs(install): fix test model repo clone commands (#53) * refactor(has_pkg): use import.metadata instead of pkg_resources (#54) * Update docs (#55) * docs: minor updates to DEVELOPER.md and README.md * docs: add docs for download utilities * docs: update docs for download utilities (#56) * docs: expand docs for MFZipFile (#58) * chore: remove empty files, fix long description in setup.cfg (#59) * fix(fixtures): fix package detection/selection (#60) * ci(release): set version to 0.1.4, update changelog Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: w-bonelli <[email protected]> Co-authored-by: Mike Taves <[email protected]>
MODFLOW-USGS · Jan 18, 2023 · ef70874 · ef70874
2 parents 614b6ee + a740101
commit ef70874
Show file tree

Hide file tree

Showing 16 changed files with 245 additions and 51 deletions.
diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -51,7 +51,7 @@ GITHUB_TOKEN=yourtoken...
 
 The tests use [`pytest-dotenv`](https://github.com/quiqua/pytest-dotenv) to detect and load variables from this file.
 
-**Note:** at minimum, the tests require that the `mf6` executable is present in `BIN_PATH`.
+**Note:** at minimum, the tests require that the `mf6` (or `mf6.exe` on Windows) executable is present in `BIN_PATH`.
 
 ### Running the tests
 

diff --git a/HISTORY.md b/HISTORY.md
@@ -1,3 +1,13 @@
+### Version 0.1.4
+
+#### Bug fixes
+
+* [fix(fixtures)](https://github.com/MODFLOW-USGS/modflow-devtools/commit/8b9aeec73885c3aa2f8bbcfa84c99824fe703cbb): Fix package detection/selection (#60). Committed by w-bonelli on 2023-01-18.
+
+#### Refactoring
+
+* [refactor(has_pkg)](https://github.com/MODFLOW-USGS/modflow-devtools/commit/861fa80f236bb9fcfcf4cfb1e9a391ad33076060): Use import.metadata instead of pkg_resources (#54). Committed by Mike Taves on 2023-01-09.
+
 ### Version 0.1.3
 
 #### Bug fixes

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # MODFLOW developer tools
 
-### Version 0.1.3 &mdash; release candidate
+### Version 0.1.4 &mdash; release candidate
 [![GitHub tag](https://img.shields.io/github/tag/MODFLOW-USGS/modflow-devtools.svg)](https://github.com/MODFLOW-USGS/modflow-devtools/tags/latest)
 [![PyPI Version](https://img.shields.io/pypi/v/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools)
 [![PyPI Versions](https://img.shields.io/pypi/pyversions/modflow-devtools.png)](https://pypi.python.org/pypi/modflow-devtools)
@@ -52,7 +52,8 @@ To install from source and set up a development environment please see the [deve
 
 This package contains shared tools for developing and testing MODFLOW 6 and FloPy, including standalone utilities as well as `pytest` fixtures, CLI options, and test cases:
 
-- a `ZipFile` subclass preserving file permissions ([more information here](https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries))
+- utilities for retrieving release information and downloading assets from the GitHub API
+- a `ZipFile` subclass that [preserves file permissions](https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries) (workaround for [Python #15795](https://bugs.python.org/issue15795))
 - a `pytest` CLI option for smoke testing (running a fast subset of cases)
 - a minimal `pytest-cases` framework for reusing test functions and data
 - a set of keepable `pytest` temporary directory fixtures for each scope

diff --git a/docs/conf.py b/docs/conf.py
@@ -8,7 +8,7 @@
 
 project = "modflow-devtools"
 author = "MODFLOW Team"
-release = "0.1.3"
+release = "0.1.4"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

diff --git a/docs/md/download.md b/docs/md/download.md
@@ -1 +1,43 @@
-# Downloads
+# Downloads
+
+Some utility functions are provided to query information and download artifacts and assets from the GitHub API. These are available in the `modflow_devtools.download` module.
+
+**Note:** to avoid GitHub API rate limits when using these functions, it is recommended to set the `GITHUB_TOKEN` environment variable. If this variable is set, the token will automatically be borne on requests sent to the API.
+
+## Retrieving information
+
+The following functions ask the GitHub API for information about a repository. The singular functions return a dictionary, while the plural functions return a list of dictionaries, with dictionary contents parsed directly from the API response's JSON.
+
+- `get_releases(repo, per_page=None, quiet=False)`
+- `get_release(repo, tag="latest", quiet=False)`
+- `list_release_assets(repo, tag="latest", quiet=False)`
+- `list_artifacts(repo, name, per_page=None, max_pages=None, quiet=False)`
+
+The `repo` parameter's format is `owner/name`, as in GitHub URLs.
+
+For instance, to download information about a release and inspect available assets:
+
+```python
+from modflow_devtools.download import get_release
+
+release = get_release("MODFLOW-USGS/executables")
+assets = release["assets"]
+expected_names = ["linux.zip", "mac.zip", "win64.zip"]
+actual_names = [asset["name"] for asset in assets]
+assert set(expected_names) == set(actual_names)
+```
+
+## Downloading assets
+
+The `download_artifact(repo, id, path=None, delete_zip=True, quiet=False)` function downloads and unzips the GitHub Actions artifact with the given ID to the given path, optionally deleting the zipfile afterwards. The `repo` format is `owner/name`, as in GitHub URLs.
+
+The `download_and_unzip(url, path=None, delete_zip=True, verbose=False)` function is a more generic alternative for downloading and unzipping files from arbitrary URLs.
+
+For instance, to download a MODFLOW 6.3.0 Linux distribution and delete the zipfile after extracting:
+
+```python
+from modflow_devtools.download import download_and_unzip
+
+url = f"https://github.com/MODFLOW-USGS/modflow6/releases/download/6.3.0/mf6.3.0_linux.zip"
+download_and_unzip(url, "some/download/path", delete_zip=True, verbose=True)
+```
diff --git a/docs/md/fixtures.md b/docs/md/fixtures.md
@@ -55,15 +55,21 @@ These models can be requested like any other `pytest` fixture, by adding one of
 - `large_test_model`: a `Path` to a large MODFLOW 6 model namefile, loaded from the `modflow6-largetestmodels` repository
 - `example_scenario`: a `Tuple[str, List[Path]]` containing the name of a MODFLOW 6 example scenario and a list of paths to its model namefiles, loaded from the `modflow6-examples` repository
 
-### Configuration
+See the [installation docs](install.md) for more information on installing test model repositories.
 
-Model repositories must first be cloned
+### Configuration
 
 It is recommended to set the environment variable `REPOS_PATH` to the location of the model repositories on the filesystem. Model repositories must live side-by-side in this location, and repository directories are expected to be named identically to GitHub repositories. If `REPOS_PATH` is not configured, `modflow-devtools` assumes tests are being run from an `autotest` subdirectory of the consuming project's root, and model repos live side-by-side with the consuming project. If this guess is incorrect and repositories cannot be found, tests requesting these fixtures will be skipped.
 
-**Note:** by default, all models found in the respective external repository will be returned by these fixtures. It is up to the consuming project to exclude models if needed.
+**Note:** by default, all models found in the respective external repository will be returned by these fixtures. It is up to the consuming project to exclude models if needed. This can be accomplished via:
+
+- custom markers
+- [filtering with CLI options](#filtering)
+- [using model-finding utility functions directly](#utility-functions)
+
+### Usage
 
-### MODFLOW 2005 test models
+#### MODFLOW 2005 test models
 
 The `test_model_mf5to6` fixture are each a `Path` to the model's namefile. For example, to load `mf5to6` models from the `MODFLOW-USGS/modflow6-testmodels` repo:
 
@@ -76,7 +82,7 @@ def test_mf5to6_model(test_model_mf5to6):
 
 This test function will be parametrized with all models found in the `mf5to6` subdirectory of the [`MODFLOW-USGS/modflow6-testmodels`](https://github.com/MODFLOW-USGS/modflow6-testmodels) repository. Note that MODFLOW-2005 namefiles need not be named `mfsim.nam`.
 
-### MODFLOW 6 test models
+#### MODFLOW 6 test models
 
 The `test_model_mf6` fixture loads all MODFLOW 6 models found in the `mf6` subdirectory of the `MODFLOW-USGS/modflow6-testmodels` repository.
 
@@ -89,7 +95,7 @@ def test_test_model_mf6(test_model_mf6):
 
 Because these are MODFLOW 6 models, each namefile will be named `mfsim.nam`. The model name can be inferred from the namefile's parent directory.
 
-### Large test models
+#### Large test models
 
 The `large_test_model` fixture loads all MODFLOW 6 models found in the `MODFLOW-USGS/modflow6-largetestmodels` repository.
 
@@ -101,7 +107,7 @@ def test_large_test_model(large_test_model):
     assert large_test_model.name == "mfsim.nam"
 ```
 
-### Example scenarios
+#### Example scenarios
 
 The [`MODFLOW-USGS/modflow6-examples`](https://github.com/MODFLOW-USGS/modflow6-examples) repository contains a collection of example scenarios, each with 1 or more models. The `example_scenario` fixture is a `Tuple[str, List[Path]]`. The first item is the name of the scenario. The second item is a list of MODFLOW 6 namefile `Path`s, ordered alphabetically by name, with models generally named as follows:
 
@@ -122,6 +128,62 @@ def test_example_scenario(tmp_path, example_scenario):
 
 **Note**: example models must first be built by running the `ci_build_files.py` script in `modflow6-examples/etc` before running tests using the `example_scenario` fixture. See the [install docs](https://modflow-devtools.readthedocs.io/en/latest/md/install.html) for more info.
 
+
+### Filtering
+
+External model test cases can be filtered by model name or by the packages the model uses with the `--model` and `--package` command line arguments, respectively.
+
+#### Filtering by model name
+
+Filtering models by name is functionally equivalent to filtering `pytest` cases with `-k`. (In the former case the filter is applied before test collection, while the latter collects tests as usual and then applies the filter.)
+
+With no filtering, collecting models from the `modflow6-largetestmodels` repo:
+
+```shell
+autotest % pytest -v test_z03_largetestmodels.py --collect-only
+...
+collected 18 items
+```
+
+Selecting a particular model by name:
+
+```shell
+autotest % pytest -v test_z03_largetestmodels.py --collect-only --model test1002_biscqtg_disv_gnc_nr_dev
+...
+collected 1 item
+
+<Module test_z03_largetestmodels.py>
+  <Function test_model[/path/to/modflow6-largetestmodels/test1002_biscqtg_disv_gnc_nr_dev/mfsim.nam]>
+```
+
+Equivalently:
+
+```shell
+autotest % pytest -v test_z03_largetestmodels.py --collect-only -k test1002_biscqtg_disv_gnc_nr_dev
+...
+collected 18 items / 17 deselected / 1 selected
+
+<Module test_z03_largetestmodels.py>
+  <Function test_model[/path/to/modflow6-largetestmodels/test1002_biscqtg_disv_gnc_nr_dev/mfsim.nam]>
+```
+
+The `--model` option can be used multiple times, e.g. `--model <model 1> --model <model 2>`.
+
+#### Filtering by package
+
+MODFLOW 6 models from external repos can also be filtered by packages used. For instance, to select only large GWT models:
+
+```shell
+autotest % pytest -v test_z03_largetestmodels.py --collect-only --package gwt
+...
+collected 3 items
+
+<Module test_z03_largetestmodels.py>
+  <Function test_model[/path/to/modflow6-largetestmodels/test1200_gwtbuy-goswami/mfsim.nam]>
+  <Function test_model[/path/to/modflow6-largetestmodels/test1201_gwtbuy-elderRa60/mfsim.nam]>
+  <Function test_model[/path/to/modflow6-largetestmodels/test2001_gwtbuy-elderRa400/mfsim.nam]>
+```
+
 ### Utility functions
 
 Model-loading fixtures use a set of utility functions to find and enumerate models. These functions can be imported from `modflow_devtools.misc` for use in other contexts:

diff --git a/docs/md/install.md b/docs/md/install.md
@@ -48,16 +48,16 @@ By default, these fixtures expect model repositories to live next to (i.e. in th
 The test model repos can simply be cloned &mdash; ideally, into the parent directory of the `modflow6` repository, so that repositories live side-by-side:
 
 ```shell
-git clone MODFLOW-USGS/modflow6-testmodels
-git clone MODFLOW-USGS/modflow6-largetestmodels
+git clone https://github.com/MODFLOW-USGS/modflow6-testmodels.git
+git clone https://github.com/MODFLOW-USGS/modflow6-largetestmodels.git
 ```
 
 ### Installing example models
 
 First clone the example models repo:
 
 ```shell
-git clone MODFLOW-USGS/modflow6-examples
+git clone https://github.com/MODFLOW-USGS/modflow6-examples.git
 ```
 
 The example models require some setup after cloning. Some extra Python dependencies are required to build the examples: 
@@ -73,4 +73,4 @@ Then, still from the `etc` folder, run:
 python ci_build_files.py
 ```
 
-This will build the examples for subsequent use by the tests.
+This will build the examples for subsequent use by the tests.
diff --git a/docs/md/zip.md b/docs/md/zip.md
@@ -1,3 +1,3 @@
 # `MFZipFile`
 
-Python's `ZipFile` doesn't preserve file permissions at extraction time. The `MFZipFile` subclass modifies `ZipFile.extract()` to do so, as per the recommendation [here](https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries).
+Python's [`ZipFile`](https://docs.python.org/3/library/zipfile.html) doesn't [preserve file permissions at extraction time](https://bugs.python.org/issue15795). The `MFZipFile` subclass modifies `ZipFile.extract()` to do so, as per the recommendation [here](https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries), and maintains identical behavior otherwise.
diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py
@@ -1,6 +1,6 @@
 __author__ = "Joseph D. Hughes"
-__date__ = "Jan 07, 2023"
-__version__ = "0.1.3"
+__date__ = "Jan 18, 2023"
+__version__ = "0.1.4"
 __maintainer__ = "Joseph D. Hughes"
 __email__ = "[email protected]"
 __status__ = "Production"

diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py
@@ -1,7 +1,9 @@
 import importlib
 import socket
 import sys
+import traceback
 from contextlib import contextmanager
+from importlib import metadata
 from os import PathLike, chdir, environ, getcwd
 from os.path import basename, normpath
 from pathlib import Path
@@ -10,7 +12,6 @@
 from typing import List, Optional, Tuple
 from urllib import request
 
-import pkg_resources
 from _warnings import warn
 
 
@@ -118,20 +119,46 @@ def get_current_branch() -> str:
 
 def get_packages(namefile_path: PathLike) -> List[str]:
     """
-    Return a list of packages used by the model defined in the given namefile.
+    Return a list of packages used by the simulation or model defined in the given namefile.
+    The namefile may be for an entire simulation or for a GWF or GWT model. If a simulation
+    namefile is given, packages used in its component model namefiles will be included.
 
     Parameters
     ----------
     namefile_path : PathLike
-        path to MODFLOW 6 name file
+        path to MODFLOW 6 simulation or model name file
     Returns
     -------
-        list of package types
+        a list of packages used by the simulation or model
     """
-    with open(namefile_path, "r") as f:
-        lines = f.readlines()
 
-    ftypes = []
+    packages = []
+    path = Path(namefile_path).expanduser().absolute()
+    lines = open(path, "r").readlines()
+    gwf_lines = [l for l in lines if l.strip().lower().startswith("gwf6 ")]
+    gwt_lines = [l for l in lines if l.strip().lower().startswith("gwt6 ")]
+
+    def parse_model_namefile(line):
+        nf_path = [path.parent / s for s in line.split(" ") if s != ""][1]
+        if nf_path.suffix != ".nam":
+            raise ValueError(
+                f"Failed to parse GWF or GWT model namefile from simulation namefile line: {line}"
+            )
+        return nf_path
+
+    # load model namefiles
+    try:
+        for line in gwf_lines:
+            packages = (
+                packages + get_packages(parse_model_namefile(line)) + ["gwf"]
+            )
+        for line in gwt_lines:
+            packages = (
+                packages + get_packages(parse_model_namefile(line)) + ["gwt"]
+            )
+    except:
+        warn(f"Invalid namefile format: {traceback.format_exc()}")
+
     for line in lines:
         # Skip over blank and commented lines
         ll = line.strip().split()
@@ -149,9 +176,9 @@ def get_packages(namefile_path: PathLike) -> List[str]:
         # strip "6" from package name
         l = l.replace("6", "")
 
-        ftypes.append(l.lower())
+        packages.append(l.lower())
 
-    return list(set(ftypes))
+    return list(set(packages))
 
 
 def has_package(namefile_path: PathLike, package: str) -> bool:
@@ -179,7 +206,7 @@ def get_namefile_paths(
     if not Path(path).is_dir():
         return []
 
-    # find namefiles
+    # find simulation namefiles
     paths = [
         p
         for p in Path(path).rglob(
@@ -310,16 +337,14 @@ def has_pkg(pkg):
     Originally written by Mike Toews ([email protected]) for FloPy.
     """
     if pkg not in _has_pkg_cache:
-
-        # for some dependencies, package name and import name are different
-        # (e.g. pyshp/shapefile, mfpymake/pymake, python-dateutil/dateutil)
-        # pkg_resources expects package name, importlib expects import name
-        try:
-            _has_pkg_cache[pkg] = bool(importlib.import_module(pkg))
-        except (ImportError, ModuleNotFoundError):
-            try:
-                _has_pkg_cache[pkg] = bool(pkg_resources.get_distribution(pkg))
-            except pkg_resources.DistributionNotFound:
-                _has_pkg_cache[pkg] = False
+        found = True
+        try:  # package name, e.g. pyshp
+            metadata.distribution(pkg)
+        except metadata.PackageNotFoundError:
+            try:  # import name, e.g. "import shapefile"
+                importlib.import_module(pkg)
+            except ModuleNotFoundError:
+                found = False
+        _has_pkg_cache[pkg] = found
 
     return _has_pkg_cache[pkg]
diff --git a/modflow_devtools/test/test_context.py b/modflow_devtools/test/test_context.py
diff --git a/modflow_devtools/test/test_framework.py b/modflow_devtools/test/test_framework.py