From 9091abec0fb82aa67bb754cad5689de6421ac031 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sat, 17 Feb 2024 22:43:56 -0500 Subject: [PATCH] wip: use pooch instead of devtools for downloads in generate_classes.py, use devtools ostags in get_modflow.py, support apple silicon releases --- autotest/test_get_modflow.py | 10 ++++++++- etc/environment.yml | 3 ++- flopy/mf6/utils/generate_classes.py | 34 ++++++++++++---------------- flopy/utils/get_modflow.py | 35 +++++++---------------------- pyproject.toml | 3 ++- 5 files changed, 35 insertions(+), 50 deletions(-) diff --git a/autotest/test_get_modflow.py b/autotest/test_get_modflow.py index ca161a04d7..142789173c 100644 --- a/autotest/test_get_modflow.py +++ b/autotest/test_get_modflow.py @@ -120,15 +120,23 @@ def test_get_release(repo): release = get_release(repo=repo, tag=tag) assets = release["assets"] + # if mac arm build is ever added for executables repo, remove + # the hacks below and add macarm.zip here expected_assets = ["linux.zip", "mac.zip", "win64.zip"] expected_ostags = [a.replace(".zip", "") for a in expected_assets] actual_assets = [asset["name"] for asset in assets] if repo == "modflow6": - # can remove if modflow6 releases follow asset name conventions followed in executables and nightly build repos + # todo: uncomment when mf6 ships apple silicon support + # expected_assets += "macarm.zip" + + # can be removed if modflow6 releases follow the same asset naming conventions as + # executables and nightly build repos assert {a.rpartition("_")[2] for a in actual_assets} >= { a for a in expected_assets if not a.startswith("win") } + elif repo == "modflow6-nightly-build": + expected_assets += "macarm.zip" else: for ostag in expected_ostags: assert any( diff --git a/etc/environment.yml b/etc/environment.yml index 58bc7e8165..7b8f514af6 100644 --- a/etc/environment.yml +++ b/etc/environment.yml @@ -8,6 +8,8 @@ dependencies: - python>=3.8 - numpy>=1.15.0,<2.0.0 - matplotlib>=1.4.0 + - pandas>=2.0.0 + - pooch # lint - black @@ -33,7 +35,6 @@ dependencies: # optional - affine - scipy - - pandas - netcdf4 - pyshp - rasterio diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index 32c1d6978c..d1bc15540d 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -2,8 +2,11 @@ import shutil import tempfile import time +from pathlib import Path from warnings import warn +from pooch import retrieve, Unzip + from .createpackages import create_packages thisfilepath = os.path.dirname(os.path.abspath(__file__)) @@ -52,29 +55,20 @@ def list_files(pth, exts=["py"]): def download_dfn(owner, repo, ref, new_dfn_pth): - try: - from modflow_devtools.download import download_and_unzip - except ImportError: - raise ImportError( - "The modflow-devtools package must be installed in order to " - "generate the MODFLOW 6 classes. This can be with:\n" - " pip install modflow-devtools" - ) - - mf6url = f"https://github.com/{owner}/{repo}/archive/{ref}.zip" - print(f" Downloading MODFLOW 6 repository from {mf6url}") with tempfile.TemporaryDirectory() as tmpdirname: - dl_path = download_and_unzip(mf6url, tmpdirname, verbose=True) - dl_contents = list(dl_path.glob("modflow6-*")) - proj_path = next(iter(dl_contents), None) - if not proj_path: - raise ValueError( - f"Could not find modflow6 project dir in {dl_path}: found {dl_contents}" + url = f"https://github.com/{owner}/{repo}/archive/{ref}.zip" + print(f" Downloading MODFLOW 6 repository from {url}") + retrieve(url, known_hash=None, path=tmpdirname, processor=Unzip()) + dl_path = Path(tmpdirname) + dl_files = list(dl_path.glob("modflow6-*")) + proj_root = next(iter(dl_files), None) + if not proj_root: + raise FileNotFoundError( + f"Could not find modflow6 project dir in {dl_path}: found {dl_files}" ) - downloaded_dfn_pth = os.path.join( - proj_path, "doc", "mf6io", "mf6ivar", "dfn" + shutil.copytree( + Path(proj_root) / "doc" / "mf6io" / "mf6ivar" / "dfn", new_dfn_pth ) - shutil.copytree(downloaded_dfn_pth, new_dfn_pth) def backup_existing_dfns(flopy_dfn_path): diff --git a/flopy/utils/get_modflow.py b/flopy/utils/get_modflow.py index 54c2a7d12c..8a304b539e 100755 --- a/flopy/utils/get_modflow.py +++ b/flopy/utils/get_modflow.py @@ -19,6 +19,12 @@ from importlib.util import find_spec from pathlib import Path +from modflow_devtools.ostags import ( + SUPPORTED_OSTAGS, + get_binary_suffixes, + get_ostag, +) + __all__ = ["run_main"] __license__ = "CC0" @@ -33,7 +39,6 @@ "modflow6-nightly-build": "modflow6_nightly", } available_repos = list(renamed_prefix.keys()) -available_ostags = ["linux", "mac", "win32", "win64"] max_http_tries = 3 # Check if this is running from flopy @@ -53,30 +58,6 @@ ) -def get_ostag() -> str: - """Determine operating system tag from sys.platform.""" - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform.startswith("win"): - return "win" + ("64" if sys.maxsize > 2**32 else "32") - elif sys.platform.startswith("darwin"): - return "mac" - raise ValueError(f"platform {sys.platform!r} not supported") - - -def get_suffixes(ostag) -> Tuple[str, str]: - if ostag in ["win32", "win64"]: - return ".exe", ".dll" - elif ostag == "linux": - return "", ".so" - elif ostag == "mac": - return "", ".dylib" - else: - raise KeyError( - f"unrecognized ostag {ostag!r}; choose one of {available_ostags}" - ) - - def get_request(url, params={}): """Get urllib.request.Request, with parameters and headers. @@ -373,7 +354,7 @@ def run_main( if ostag is None: ostag = get_ostag() - exe_suffix, lib_suffix = get_suffixes(ostag) + exe_suffix, lib_suffix = get_binary_suffixes(ostag) # select bindir if path not provided if bindir.startswith(":"): @@ -712,7 +693,7 @@ def cli_main(): ) parser.add_argument( "--ostag", - choices=available_ostags, + choices=SUPPORTED_OSTAGS, help="Operating system tag; default is to automatically choose.", ) parser.add_argument( diff --git a/pyproject.toml b/pyproject.toml index 6724591869..a958a4f8c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,8 @@ requires-python = ">=3.8" dependencies = [ "numpy >=1.15.0,<2.0.0", "matplotlib >=1.4.0", - "pandas >=2.0.0" + "pandas >=2.0.0", + "pooch" ] dynamic = ["version", "readme"]