refactor(dist scripts): use pathlib, find paths relative to script lo…

…cn, use matrix for nightly build test CI

.github/workflows/ci-check-warnings-gfortran.yml

@@ -20,21 +20,21 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v2.3.4
+        uses: actions/checkout@v3
 
       - name: Install Conda environment from environment.yml
         uses: mamba-org/provision-with-micromamba@main
         with:
-            cache-downloads: true
-            cache-env: true
-
-      - name: Install gfortran
-        uses: modflowpy/install-gfortran-action@v1
+          cache-downloads: true
+          cache-env: true
 
       - name: Print python package versions
         run: |
           pip list
 
+      - name: Install gfortran
+        uses: modflowpy/install-gfortran-action@v1
+
       - name: Setup modflow
         run: |
           meson setup builddir -Ddebug=false -Dwerror=true
@@ -44,28 +44,38 @@ jobs:
           meson compile -C builddir
 
   test_night_build_script:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            ostag: linux
+          - os: macos-latest
+            ostag: mac
+          - os: windows-latest
+            ostag: win64
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v2.3.4
+        uses: actions/checkout@v3
 
       - name: Install Conda environment from environment.yml
         uses: mamba-org/provision-with-micromamba@main
         with:
-            cache-downloads: true
-            cache-env: true
-
-      - name: Install gfortran
-        uses: modflowpy/install-gfortran-action@v1
+          cache-downloads: true
+          cache-env: true
 
       - name: Print python package versions
         run: |
           pip list
 
+      - name: Setup gfortran
+        uses: modflowpy/install-gfortran-action@v1
+
       - name: Update flopy MODFLOW 6 classes
         working-directory: autotest
         run: |
@@ -74,10 +84,10 @@ jobs:
       - name: Run nightly build script
         working-directory: distribution
         run: |
-          pytest -v -s build_nightly.py
+          python build_nightly.py
 
-      - name: Move the build zip file
+      - name: Make sure zip file exists
+        working-directory: distribution
         run: |
-          ls -l ./distribution/*
-          mv ./distribution/temp_zip/linux.zip ./linux.zip
-          ls -l ./
+          path="temp_zip/${{ matrix.ostag }}.zip"
+          [ -f "$path" ] && echo "Zipfile created: $path" || echo "Zipfile not found: $path"; exit 1

distribution/build_nightly.py

@@ -1,15 +1,15 @@
-import os
-import pathlib
+from os import PathLike, chdir, X_OK
+from pathlib import Path
 import platform
-import shutil
+from shutil import which
+import subprocess
 import sys
 
 import flopy
 import pymake
 
-# add path to build script in autotest directory and reuse mf6 build scripts
-sys.path.append(os.path.join("..", "autotest"))
-from build_exes import meson_build
+from make_release import update_version
+from mkdist import update_mf6io_tex_files
 
 # make sure exe extension is used on windows
 eext = ""
@@ -18,10 +18,6 @@
     eext = ".exe"
     soext = ".dll"
 
-bin_path = os.path.abspath(os.path.join("..", "bin"))
-example_path = os.path.abspath(os.path.join("temp"))
-zip_path = os.path.abspath(os.path.join("temp_zip"))
-
 
 def get_zipname():
     zipname = sys.platform.lower()
@@ -35,78 +31,58 @@ def get_zipname():
     return zipname
 
 
-def relpath_fallback(pth):
-    try:
-        # throws ValueError on Windows if pth is on a different drive
-        return os.path.relpath(pth)
-    except ValueError:
-        return os.path.abspath(pth)
-
-
-def create_dir(pth):
-    # remove pth directory if it exists
-    if os.path.exists(pth):
-        print(f"removing... {os.path.abspath(pth)}")
-        shutil.rmtree(pth)
-
-    # create pth directory
-    print(f"creating... {os.path.abspath(pth)}")
-    os.makedirs(pth)
-
-    msg = f"could not create... {os.path.abspath(pth)}"
-    assert os.path.exists(pth), msg
-
-
-def test_update_version():
-    from make_release import update_version
-
-    update_version()
-
-
-def test_create_dirs():
-    for pth in (
-        bin_path,
-        zip_path,
-    ):
-        create_dir(pth)
+def create_nightly_build(workspace_path: PathLike, zip_path: PathLike):
+    chdir(Path(workspace_path))
 
+    # build release source files with Meson
+    cmd = (
+            f"meson setup builddir "
+            + f"--bindir=bin "
+            + f"--libdir=bin "
+            + f"--prefix={('%CD%' if platform.system() == 'Windows' else '$(pwd)')}"
+            + (" --wipe" if Path("builddir").is_dir() else "")
+    )
+    print(f"Running meson setup command: {cmd}")
+    subprocess.run(cmd, shell=True, check=True)
 
-def test_nightly_build():
-    meson_build()
+    cmd = "meson install -C builddir"
+    print(f"Running meson install command: {cmd}")
+    subprocess.run(cmd, shell=True, check=True)
 
     # test if there are any executable files to zip
     binpth_files = [
-        os.path.join(bin_path, f)
-        for f in os.listdir(bin_path)
-        if os.path.isfile(os.path.join(bin_path, f))
-        and shutil.which(os.path.join(bin_path, f), mode=os.X_OK)
-        and pathlib.Path(os.path.join(bin_path, f)).suffix
-        not in (".a", ".lib", ".pdb")
+        str(bin_path / f)
+        for f in bin_path.glob('*')
+        if (bin_path / f).is_file()
+           and which(bin_path / f, mode=X_OK)
+           and (bin_path / f).suffix not in (".a", ".lib", ".pdb")
     ]
     if len(binpth_files) < 1:
         raise FileNotFoundError(
-            f"No executable files present in {os.path.abspath(bin_path)}.\n"
-            + f"Available files:\n [{', '.join(os.listdir(bin_path))}]"
+            f"No executable files present in {bin_path.absolute()}.\n"
+            + f"Available files:\n [{', '.join([str(p) for p in bin_path.glob('*')])}]"
         )
     else:
         print(f"Files to zip:\n [{', '.join(binpth_files)}]")
 
-    zip_pth = os.path.abspath(os.path.join(zip_path, get_zipname() + ".zip"))
-    print(f"Zipping files to '{zip_pth}'")
-    success = pymake.zip_all(zip_pth, file_pths=binpth_files)
-    assert success, f"Could not create '{zip_pth}'"
+    zip_path = (Path(zip_path) / (get_zipname() + ".zip")).absolute()
+    print(f"Zipping files to '{zip_path}'")
+    success = pymake.zip_all(str(zip_path), file_pths=binpth_files)
+    assert success, f"Could not create '{zip_path}'"
 
 
-def test_update_mf6io():
-    from mkdist import update_mf6io_tex_files
+def update_mf6io(bin_path: PathLike, example_path: PathLike):
+    bin_path = Path(bin_path)
+    example_path = Path(example_path)
 
-    # build simple model
     name = "mymodel"
-    ws = os.path.join(example_path, name)
+    ws = str(example_path / name)
     exe_name = "mf6"
     if sys.platform.lower() == "win32":
         exe_name += ".exe"
-    exe_name = os.path.join(bin_path, exe_name)
+    exe_name = str(bin_path / exe_name)
+
+    # build simple model
     sim = flopy.mf6.MFSimulation(sim_name=name, sim_ws=ws, exe_name=exe_name)
     tdis = flopy.mf6.ModflowTdis(sim)
     ims = flopy.mf6.ModflowIms(sim)
@@ -125,7 +101,16 @@ def test_update_mf6io():
 
 
 if __name__ == "__main__":
-    test_update_version()
-    test_create_dirs()
-    test_nightly_build()
-    test_update_mf6io()
+    build_path = Path(__file__).parent.parent
+    bin_path = build_path / "bin"
+    example_path = build_path / "temp"
+    zip_path = build_path / "distribution" / "temp_zip"
+
+    print(f"Building nightly MODFLOW 6 release")
+    bin_path.mkdir(exist_ok=True, parents=True)
+    example_path.mkdir(exist_ok=True, parents=True)
+    zip_path.mkdir(exist_ok=True, parents=True)
+
+    update_version()
+    create_nightly_build(build_path, zip_path)
+    update_mf6io(bin_path, example_path)

distribution/make_release.py

@@ -53,11 +53,10 @@
 import subprocess
 import sys
 from collections import OrderedDict
+from pathlib import Path
 
-# update files and paths so that there are the same number of
-# path and file entries in the paths and files list. Enter '.'
-# as the path if the file is in the root repository directory
-paths = ["../", "../doc", "../", "../", "../", "../src/Utilities"]
+project_root = Path(__file__).parent.parent
+paths = [project_root, project_root / "doc", project_root, project_root, project_root, project_root / "src" / "Utilities"]
 files = [
     "version.txt",
     "version.tex",
@@ -158,7 +157,7 @@ def get_is_approved():
     if is_approved is None:
         # get current branch
         branch = get_branch()
-        if "release" in branch.lower() or "master" in branch.lower():
+        if branch is None or "release" in branch.lower() or "master" in branch.lower():
             is_approved = True
         else:
             is_approved = False
@@ -249,7 +248,7 @@ def update_version():
             vmicro = int(t[2])
 
     try:
-        fpth = os.path.join(paths[0], files[0])
+        fpth = str(paths[0] / files[0])
         lines = [line.rstrip("\n") for line in open(fpth, "r")]
         if vmajor is None:
             for line in lines:
@@ -283,8 +282,8 @@ def update_version():
         # update version.py in doc directory
         shutil.copyfile(
             os.path.abspath(fpth),
-            os.path.join(
-                "..", "doc", os.path.basename(fpth.replace(".txt", ".py"))
+            str(
+                project_root / "doc" / os.path.basename(fpth.replace(".txt", ".py"))
             ),
         )
 
@@ -293,7 +292,7 @@ def update_version():
         version_type = get_version_type(get_branch()).strip()
         if len(version_type) > 0:
             version += f"---{version_type}"
-        pth = os.path.join(paths[1], files[1])
+        pth = str(paths[1] / files[1])
         f = open(pth, "w")
         line = "\\newcommand{\\modflowversion}{mf" + f"{version}" + "}"
         f.write(f"{line}\n")
@@ -381,7 +380,7 @@ def update_mf6_version(vmajor, vminor, vmicro):
     is_approved, disclaimerfmt = get_disclaimerfmt()
 
     # read version.f90 into memory
-    fpth = os.path.join(paths[5], files[5])
+    fpth = str(paths[5] / files[5])
     with open(fpth, "r") as file:
         lines = [line.rstrip() for line in file]
 
@@ -429,7 +428,7 @@ def update_readme_markdown(vmajor, vminor, vmicro):
         sb = " release candidate"
 
     # read README.md into memory
-    fpth = os.path.join(paths[2], files[2])
+    fpth = str(paths[2] / files[2])
     with open(fpth, "r") as file:
         lines = [line.rstrip() for line in file]
 
@@ -439,7 +438,7 @@ def update_readme_markdown(vmajor, vminor, vmicro):
     for line in lines:
         if "## Version " in line:
             line = f"### Version {version}"
-            if "develop" in branch:
+            if branch is not None and "develop" in branch:
                 line += sb
         # This has been commented out as we've generalized this reference.
         # elif "https://doi.org/10.5066/F76Q1VQV" in line:
@@ -464,7 +463,7 @@ def update_readme_markdown(vmajor, vminor, vmicro):
     f.close()
 
     # write disclaimer markdown file
-    fpth = os.path.join(paths[3], files[3])
+    fpth = str(paths[3] / files[3])
     f = open(fpth, "w")
     f.write(disclaimer)
     f.close()
@@ -474,7 +473,7 @@ def update_readme_markdown(vmajor, vminor, vmicro):
 
 def update_codejson(vmajor, vminor, vmicro):
     # define json filename
-    json_fname = os.path.join(paths[4], files[4])
+    json_fname = str(paths[4] / files[4])
 
     # get branch
     branch = get_branch()