From 4a7adc0edcb265871fdb805d6edb0df3d49693c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 5 Aug 2023 18:12:37 +0000 Subject: [PATCH 1/4] ci(release): update to development version 1.1.0.dev0 --- docs/conf.py | 2 +- modflow_devtools/__init__.py | 2 +- version.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0048764..f780c20 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = "modflow-devtools" author = "MODFLOW Team" -release = "1.0.0" +release = "1.1.0.dev0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index ea1d042..a825793 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -1,6 +1,6 @@ __author__ = "Joseph D. Hughes" __date__ = "Aug 05, 2023" -__version__ = "1.0.0" +__version__ = "1.1.0.dev0" __maintainer__ = "Joseph D. Hughes" __email__ = "jdhughes@usgs.gov" __status__ = "Production" diff --git a/version.txt b/version.txt index afaf360..64b95b0 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.1.0.dev0 \ No newline at end of file From 582d48a4d72f18a787216ada5befb7543cebdfcf Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Tue, 8 Aug 2023 01:31:23 -0400 Subject: [PATCH 2/4] refactor: deprecate misc functions, add ostags alternatives (#105) - deprecate misc.get_ostag() for ostags.get_modflow_ostag() - deprecate misc.get_suffixes() for ostags.get_binary_suffixes() - improve some docstrings - expand tests - update docs --- docs/md/ostags.md | 11 +++++++ modflow_devtools/misc.py | 47 ++++++++++++++++++++++++---- modflow_devtools/ostags.py | 46 +++++++++++++++++++++++++-- modflow_devtools/test/test_ostags.py | 41 +++++++++++++++++++++++- 4 files changed, 135 insertions(+), 10 deletions(-) diff --git a/docs/md/ostags.md b/docs/md/ostags.md index e140d98..ea9440c 100644 --- a/docs/md/ostags.md +++ b/docs/md/ostags.md @@ -43,3 +43,14 @@ OSTag.convert(platform.system(), "py2mf") The second argument specifies the mapping in format `2`, where `` and `` may take values `py`, `mf`, or `gh`. **Note**: source and target must be different. + +## Getting suffixes + +A convenience function is available to get the appropriate binary file extensions for a given operating system, identified by tag (or the current operating system if no tag is provided). The return value is a 2-tuple containing the executable and library extensions, respectively. + +```python +get_binary_suffixes() # get extensions for current OS +get_binary_suffixes("linux") # returns ("", ".so") +get_binary_suffixes("linux") # returns ("", ".so") +get_binary_suffixes("win64") # returns (".exe", ".dll") +``` diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py index c326c0b..31d540d 100644 --- a/modflow_devtools/misc.py +++ b/modflow_devtools/misc.py @@ -82,7 +82,12 @@ def __exit__(self, exc_type, exc_value, traceback): def get_ostag() -> str: - """Determine operating system tag from sys.platform.""" + """ + Determine operating system tag from sys.platform. + + .. deprecated:: 1.1.0 + Use ``ostags.get_modflow_ostag()`` instead. + """ if sys.platform.startswith("linux"): return "linux" elif sys.platform.startswith("win"): @@ -93,7 +98,12 @@ def get_ostag() -> str: def get_suffixes(ostag) -> Tuple[str, str]: - """Returns executable and library suffixes for the given OS (as returned by sys.platform)""" + """ + Returns executable and library suffixes for the given OS (as returned by sys.platform) + + .. deprecated:: 1.1.0 + Use ``ostags.get_binary_suffixes()`` instead. + """ tag = ostag.lower() @@ -136,6 +146,23 @@ def run_py_script(script, *args, verbose=False): def get_current_branch() -> str: + """ + Tries to determine the name of the current branch, first by the GITHUB_REF + environent variable, then by asking ``git`` if GITHUB_REF is not set. + + Returns + ------- + str + name of the current branch + + Raises + ------ + RuntimeError + if ``git`` is not available + ValueError + if the current branch could not be determined + """ + # check if on GitHub Actions CI ref = environ.get("GITHUB_REF") if ref is not None: @@ -160,6 +187,7 @@ def get_packages(namefile_path: PathLike) -> List[str]: ---------- namefile_path : PathLike path to MODFLOW 6 simulation or model name file + Returns ------- a list of packages used by the simulation or model @@ -215,7 +243,9 @@ def parse_model_namefile(line): def has_package(namefile_path: PathLike, package: str) -> bool: - """Determines whether the model with the given namefile contains the selected package""" + """ + Determines whether the model with the given namefile contains the selected package. + """ packages = get_packages(namefile_path) return package.lower() in packages @@ -306,7 +336,9 @@ def get_model_paths( def is_connected(hostname): """ Tests whether the given URL is accessible. - See https://stackoverflow.com/a/20913928/.""" + See https://stackoverflow.com/a/20913928/. + """ + try: host = socket.gethostbyname(hostname) s = socket.create_connection((host, 80), 2) @@ -318,7 +350,10 @@ def is_connected(hostname): def is_in_ci(): - """Determines whether the current process is running GitHub Actions CI""" + """ + Determines whether the current process is running GitHub Actions CI + by checking for the "CI" environment variable. + """ # if running in GitHub Actions CI, "CI" variable always set to true # https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables @@ -328,7 +363,7 @@ def is_in_ci(): def is_github_rate_limited() -> Optional[bool]: """ Determines if a GitHub API rate limit is applied to the current IP. - Running this function will consume an API request! + Calling this function will consume an API request! Returns ------- diff --git a/modflow_devtools/ostags.py b/modflow_devtools/ostags.py index 55e7706..3c68ed3 100644 --- a/modflow_devtools/ostags.py +++ b/modflow_devtools/ostags.py @@ -1,18 +1,20 @@ """ -MODFLOW 6, Python3, and build servers may all refer to operating -systems by different names. This module contains conversion utilities. +MODFLOW 6, Python3, and GitHub Actions refer to operating +systems differently. This module contains conversion utilities. """ +import sys from enum import Enum from platform import system +from typing import Tuple _system = system() def get_modflow_ostag() -> str: if _system == "Windows": - return "win64" + return "win" + ("64" if sys.maxsize > 2**32 else "32") elif _system == "Linux": return "linux" elif _system == "Darwin": @@ -30,6 +32,44 @@ def get_github_ostag() -> str: raise NotImplementedError(f"Unsupported system: {_system}") +def get_binary_suffixes(ostag: str = None) -> Tuple[str, str]: + """ + Returns executable and library suffixes for the given OS tag, if provided, + otherwise for the current operating system. + + Parameters + ---------- + ostag : str, optional + The OS tag. May be provided in modflow, python, or github format. + + Returns + ------- + Tuple[str, str] + The executable and library suffixes, respectively. + """ + + if ostag is None: + ostag = get_modflow_ostag() + + def _suffixes(tag): + if tag in ["win32", "win64"]: + return ".exe", ".dll" + elif tag == "linux": + return "", ".so" + elif tag == "mac" or tag == "darwin": + return "", ".dylib" + else: + raise KeyError(f"unrecognized OS tag: {tag!r}") + + try: + return _suffixes(ostag.lower()) + except: + try: + return _suffixes(python_to_modflow_ostag(ostag)) + except: + return _suffixes(github_to_modflow_ostag(ostag)) + + def python_to_modflow_ostag(tag: str) -> str: """ Convert a platform.system() string to an ostag as expected diff --git a/modflow_devtools/test/test_ostags.py b/modflow_devtools/test/test_ostags.py index 02d9adb..7ad3b91 100644 --- a/modflow_devtools/test/test_ostags.py +++ b/modflow_devtools/test/test_ostags.py @@ -1,7 +1,12 @@ from platform import system import pytest -from modflow_devtools.ostags import OSTag, get_github_ostag, get_modflow_ostag +from modflow_devtools.ostags import ( + OSTag, + get_binary_suffixes, + get_github_ostag, + get_modflow_ostag, +) _system = system() @@ -53,3 +58,37 @@ def test_get_github_ostag(): ) def test_ostag_convert(cvt, tag, exp): assert OSTag.convert(tag, cvt) == exp + + +def test_get_binary_suffixes(): + exe, lib = get_binary_suffixes() + if _system == "Windows": + assert exe == ".exe" + assert lib == ".dll" + elif _system == "Linux": + assert exe == "" + assert lib == ".so" + elif _system == "Darwin": + assert exe == "" + assert lib == ".dylib" + + +@pytest.mark.parametrize( + "tag,exe,lib", + [ + ("win64", ".exe", ".dll"), + ("win32", ".exe", ".dll"), + ("Windows", ".exe", ".dll"), + ("linux", "", ".so"), + ("Linux", "", ".so"), + ("mac", "", ".dylib"), + ("macOS", "", ".dylib"), + ("Darwin", "", ".dylib"), + ], +) +def test_get_binary_suffixes_given_tag(tag, exe, lib): + from modflow_devtools.misc import get_suffixes + + assert get_binary_suffixes(tag) == (exe, lib) + if tag in ("win64", "win32", "linux", "mac"): + assert get_suffixes(tag) == (exe, lib) From 03ea04157190480b455e174de64c692ff3bb86a3 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Sat, 12 Aug 2023 12:47:03 -0400 Subject: [PATCH 3/4] refactor(has_pkg): introduce strict flag (#106) * add strict flag to has_pkg() toggling whether to try to import or only check metadata * always invalidate/refresh the installed package cache if strict is on * add pytest-virtualenv to test dependencies, test with/without strict * use --dist loadfile with xdist in CI for compatibility * use strict=True for requires_pkg marker --- .github/workflows/ci.yml | 3 ++- conftest.py | 3 +++ modflow_devtools/markers.py | 2 +- modflow_devtools/misc.py | 42 ++++++++++++++++++++++++------ modflow_devtools/test/test_misc.py | 34 ++++++++++++++++++++++++ pyproject.toml | 1 + 6 files changed, 75 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c18594..4738d35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,8 @@ jobs: BIN_PATH: ~/.local/bin/modflow REPOS_PATH: ${{ github.workspace }} GITHUB_TOKEN: ${{ github.token }} - run: pytest -v -n auto --durations 0 --ignore modflow_devtools/test/test_download.py + # use --dist loadfile to so tests requiring pytest-virtualenv run on the same worker + run: pytest -v -n auto --dist loadfile --durations 0 --ignore modflow_devtools/test/test_download.py - name: Run network-dependent tests # only invoke the GH API on one OS and Python version diff --git a/conftest.py b/conftest.py index 74bcb56..d952441 100644 --- a/conftest.py +++ b/conftest.py @@ -1 +1,4 @@ +from pathlib import Path + pytest_plugins = ["modflow_devtools.fixtures"] +project_root_path = Path(__file__).parent diff --git a/modflow_devtools/markers.py b/modflow_devtools/markers.py index 0aa4b25..d40ca5c 100644 --- a/modflow_devtools/markers.py +++ b/modflow_devtools/markers.py @@ -22,7 +22,7 @@ def requires_exe(*exes): def requires_pkg(*pkgs): - missing = {pkg for pkg in pkgs if not has_pkg(pkg)} + missing = {pkg for pkg in pkgs if not has_pkg(pkg, strict=True)} return pytest.mark.skipif( missing, reason=f"missing package{'s' if len(missing) != 1 else ''}: " diff --git a/modflow_devtools/misc.py b/modflow_devtools/misc.py index 31d540d..7dd9210 100644 --- a/modflow_devtools/misc.py +++ b/modflow_devtools/misc.py @@ -398,21 +398,47 @@ def has_exe(exe): return _has_exe_cache[exe] -def has_pkg(pkg): +def has_pkg(pkg: str, strict: bool = False) -> bool: """ Determines if the given Python package is installed. + Parameters + ---------- + pkg : str + Name of the package to check. + strict : bool + If False, only check if package metadata is available. + If True, try to import the package (all dependencies must be present). + + Returns + ------- + bool + True if the package is installed, otherwise False. + + Notes + ----- Originally written by Mike Toews (mwtoews@gmail.com) for FloPy. """ - if pkg not in _has_pkg_cache: - found = True + + def try_import(): + try: # import name, e.g. "import shapefile" + importlib.import_module(pkg) + return True + except ModuleNotFoundError: + return False + + def try_metadata() -> bool: try: # package name, e.g. pyshp metadata.distribution(pkg) + return True except metadata.PackageNotFoundError: - try: # import name, e.g. "import shapefile" - importlib.import_module(pkg) - except ModuleNotFoundError: - found = False - _has_pkg_cache[pkg] = found + return False + + found = False + if not strict: + found = pkg in _has_pkg_cache or try_metadata() + if not found: + found = try_import() + _has_pkg_cache[pkg] = found return _has_pkg_cache[pkg] diff --git a/modflow_devtools/test/test_misc.py b/modflow_devtools/test/test_misc.py index fe6d2cb..79ea786 100644 --- a/modflow_devtools/test/test_misc.py +++ b/modflow_devtools/test/test_misc.py @@ -2,14 +2,17 @@ import shutil from os import environ from pathlib import Path +from pprint import pprint from typing import List import pytest +from conftest import project_root_path from modflow_devtools.misc import ( get_model_paths, get_namefile_paths, get_packages, has_package, + has_pkg, set_dir, set_env, ) @@ -249,3 +252,34 @@ def test_get_namefile_paths_select_patterns(): def test_get_namefile_paths_select_packages(): paths = get_namefile_paths(_examples_path, packages=["wel"]) assert len(paths) >= 43 + + +@pytest.mark.slow +def test_has_pkg(virtualenv): + python = virtualenv.python + venv = Path(python).parent + pkg = "pytest" + dep = "pluggy" + print( + f"Using temp venv at {venv} with python {python} to test has_pkg('{pkg}') with and without '{dep}'" + ) + + # install a package and remove one of its dependencies + virtualenv.run(f"pip install {project_root_path}") + virtualenv.run(f"pip install {pkg}") + virtualenv.run(f"pip uninstall -y {dep}") + + # check with/without strict mode + for strict in [False, True]: + cmd = ( + f"from modflow_devtools.misc import has_pkg; print(has_pkg('{pkg}'" + + (", strict=True))" if strict else "))") + ) + exp = "False" if strict else "True" + assert ( + virtualenv.run( + f'{python} -c "{cmd}"', + capture=True, + ).strip() + == exp + ) diff --git a/pyproject.toml b/pyproject.toml index 3dedbcb..c3630b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ test = [ "pytest-cases", "pytest-cov", "pytest-dotenv", + "pytest-virtualenv", "pytest-xdist", "PyYaml" ] From cacd9440f1da56a7c09137532ec93757f12d2239 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 16:51:11 +0000 Subject: [PATCH 4/4] ci(release): set version to 1.1.0, update changelog --- HISTORY.md | 7 +++++++ docs/conf.py | 2 +- modflow_devtools/__init__.py | 4 ++-- version.txt | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 906d6a1..96524f3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,10 @@ +### Version 1.1.0 + +#### Refactoring + +* [refactor](https://github.com/MODFLOW-USGS/modflow-devtools/commit/582d48a4d72f18a787216ada5befb7543cebdfcf): Deprecate misc functions, add ostags alternatives (#105). Committed by w-bonelli on 2023-08-08. +* [refactor(has_pkg)](https://github.com/MODFLOW-USGS/modflow-devtools/commit/03ea04157190480b455e174de64c692ff3bb86a3): Introduce strict flag (#106). Committed by w-bonelli on 2023-08-12. + ### Version 1.0.0 #### New features diff --git a/docs/conf.py b/docs/conf.py index f780c20..dbe6721 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ project = "modflow-devtools" author = "MODFLOW Team" -release = "1.1.0.dev0" +release = "1.1.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index a825793..abd3212 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -1,6 +1,6 @@ __author__ = "Joseph D. Hughes" -__date__ = "Aug 05, 2023" -__version__ = "1.1.0.dev0" +__date__ = "Aug 12, 2023" +__version__ = "1.1.0" __maintainer__ = "Joseph D. Hughes" __email__ = "jdhughes@usgs.gov" __status__ = "Production" diff --git a/version.txt b/version.txt index 64b95b0..1cc5f65 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.1.0.dev0 \ No newline at end of file +1.1.0 \ No newline at end of file