From 9069ecbdca93c5321e62bb0a0a5518d86d8d666b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 21 Dec 2023 21:59:56 -0500 Subject: [PATCH 1/2] test: remove support for .cmp reference files * remove testmodel comparison support for /compare/*.cmp files * fix default mf6 regression executable selection * misc cleanup --- autotest/common_regression.py | 69 ++++++---------- autotest/framework.py | 147 +++++++++++----------------------- 2 files changed, 68 insertions(+), 148 deletions(-) diff --git a/autotest/common_regression.py b/autotest/common_regression.py index 0c91f95b617..b78942e8d1a 100644 --- a/autotest/common_regression.py +++ b/autotest/common_regression.py @@ -4,20 +4,14 @@ from typing import List, Tuple from warnings import warn -COMPARE_PATTERNS = ( - ".cmp", +COMPARE_PROGRAMS = ( "mf2005", - "mf2005.cmp", "mfnwt", - "mfnwt.cmp", "mfusg", - "mfusg.cmp", "mflgr", - "mflgr.cmp", "libmf6", - "libmf6.cmp", "mf6", - "mf6.cmp", + # todo: "mp7" ) IGNORE_EXTENSIONS = ( ".hds", @@ -121,8 +115,6 @@ def model_setup(namefile, dst, remove_existing=True, extrafiles=None): else: print(f"{srcf} does not exist") - return - def get_input_files(namefile): """Return a list of all the input files in this model. @@ -332,13 +324,13 @@ def get_mf6_comparison(src): # Construct src pth from namefile for _, dirs, _ in os.walk(src): dl = [d.lower() for d in dirs] - for pattern in COMPARE_PATTERNS: + for pattern in COMPARE_PROGRAMS: if any(pattern in s for s in dl): return pattern def setup_mf6_comparison( - src, dst, compare="auto", overwrite=True, verbose=False + src, dst, cmp_exe="mf6", overwrite=True, verbose=False ): """Setup comparison for MODFLOW 6 simulation. @@ -348,10 +340,8 @@ def setup_mf6_comparison( Directory with original MODFLOW 6 input files. dst : path-like Directory to copy MODFLOW 6 input files to. - compare : str, optional - Type of comparison, e.g. 'auto', 'mf6', 'mf2005'. Use "auto" to include all - files in the source folder whose name includes "cmp". If not "auto", selects - the subdirectory of `src` containing reference files (default is "auto"). + cmp_exe : str or PathLike, optional + Program to compare with, for supported see `COMPARE_PROGRAMSa. overwrite : bool, optional Whether to overwrite the destination directory if it exists (default is True). verbose : bool, optional @@ -364,12 +354,12 @@ def setup_mf6_comparison( """ - if compare is None: + if cmp_exe is None: warn(f"No action provided, aborting") return # create and/or clean dest dir if needed - dst = Path(dst) / compare + dst = Path(dst) / cmp_exe dst.mkdir(exist_ok=True) dls = list(os.walk(dst)) if overwrite and any(dls): @@ -390,38 +380,25 @@ def setup_mf6_comparison( raise ValueError(f"Destination exists but overwrite disabled: {dst}") # copy files - cmppth = os.path.join(src, compare) + cmppth = os.path.join(src, cmp_exe) files = os.listdir(cmppth) files2copy = [] - if compare.lower() == "auto": + if "mf6" in cmp_exe.lower(): for file in files: - if ".cmp" in os.path.splitext(file)[1].lower(): - files2copy.append(os.path.join(cmppth, file)) - for srcf in files2copy: - f = os.path.basename(srcf) - dstf = os.path.join(dst, f) - if os.path.exists(srcf): - print(f"Copying file '{srcf}' -> '{dstf}'") - shutil.copy(srcf, dstf) - else: - print(srcf + " does not exist") + if "mfsim.nam" in file.lower(): + srcf = os.path.join(cmppth, os.path.basename(file)) + files2copy.append(srcf) + srcdir = os.path.join(src, cmp_exe) + setup_mf6(srcdir, dst, remove_existing=overwrite) + break else: - if "mf6" in compare.lower(): - for file in files: - if "mfsim.nam" in file.lower(): - srcf = os.path.join(cmppth, os.path.basename(file)) - files2copy.append(srcf) - srcdir = os.path.join(src, compare) - setup_mf6(srcdir, dst, remove_existing=overwrite) - break - else: - for file in files: - if ".nam" in os.path.splitext(file)[1].lower(): - srcf = os.path.join(cmppth, os.path.basename(file)) - files2copy.append(srcf) - nf = os.path.join(src, compare, os.path.basename(file)) - model_setup(nf, dst, remove_existing=overwrite) - break + for file in files: + if ".nam" in os.path.splitext(file)[1].lower(): + srcf = os.path.join(cmppth, os.path.basename(file)) + files2copy.append(srcf) + nf = os.path.join(src, cmp_exe, os.path.basename(file)) + model_setup(nf, dst, remove_existing=overwrite) + break def get_mf6_files(namefile, verbose=False): diff --git a/autotest/framework.py b/autotest/framework.py index 297b80953e4..ad4d54acd1e 100644 --- a/autotest/framework.py +++ b/autotest/framework.py @@ -171,9 +171,6 @@ def write_input(*sims, verbose=True): class TestFramework: - # tell pytest this class doesn't contain tests, don't collect it - __test__ = False - """ Defines a MODFLOW 6 test and its lifecycle, with configurable hooks to evaluate results or run other models for comparison: @@ -182,8 +179,6 @@ class TestFramework: - MODFLOW-NWT - MODFLOW-USG - MODFLOW-LGR - - MODPATH 6 - - MODPATH 7 Parameters ---------- @@ -226,7 +221,8 @@ class TestFramework: 'mf6_regression', or 'run_only' """ - # builtin + # tell pytest this class doesn't contain tests, don't collect it + __test__ = False def __init__( self, @@ -283,24 +279,15 @@ def __repr__(self): # private def _compare_heads( - self, compare=None, cpth=None, extensions="hds" + self, cpth=None, extensions="hds" ) -> bool: if isinstance(extensions, str): extensions = [extensions] - # if a comparison path is provided, compare with the reference model results if cpth: - files_cmp = None - if compare is not None and compare.lower() == "auto": - files_cmp = [] - files = os.listdir(cpth) - for file in files: - files_cmp.append(file) - files1 = [] files2 = [] exfiles = [] - ipos = 0 for file1 in self.outp: ext = os.path.splitext(file1)[1][1:] @@ -317,54 +304,38 @@ def _compare_heads( exfiles.append(None) # Check to see if there is a corresponding compare file - if files_cmp is not None: - if file1 + ".cmp" in files_cmp: - # compare file - idx = files_cmp.index(file1 + ".cmp") - pth = os.path.join(cpth, files_cmp[idx]) - files2.append(pth) - print( - f"Comparison file {ipos + 1}", - os.path.basename(pth), - ) + if self.coutp is not None: + for file2 in self.coutp: + ext = os.path.splitext(file2)[1][1:] + if ext.lower() in extensions: + # simulation file + pth = os.path.join(cpth, file2) + files2.append(pth) else: - if self.coutp is not None: - for file2 in self.coutp: - ext = os.path.splitext(file2)[1][1:] - - if ext.lower() in extensions: - # simulation file - pth = os.path.join(cpth, file2) - files2.append(pth) - - else: - files2.append(None) + files2.append(None) if self.cmp_namefile is None: pth = None else: pth = os.path.join(cpth, self.cmp_namefile) - for ipos in range(len(files1)): - file1 = files1[ipos] + for i in range(len(files1)): + file1 = files1[i] ext = os.path.splitext(file1)[1][1:].lower() outfile = os.path.splitext(os.path.basename(file1))[0] outfile = os.path.join( self.workspace, outfile + "." + ext + ".cmp.out" ) - if files2 is None: - file2 = None - else: - file2 = files2[ipos] + file2 = None if files2 is None else files2[i] # set exfile exfile = None if file2 is None: if len(exfiles) > 0: - exfile = exfiles[ipos] + exfile = exfiles[i] if exfile is not None: print( - f"Exclusion file {ipos + 1}", + f"Exclusion file {i + 1}", os.path.basename(exfile), ) @@ -379,23 +350,18 @@ def _compare_heads( files2=file2, htol=self.htol, difftol=True, - # Change to true to have list of all nodes exceeding htol verbose=self.verbose, exfile=exfile, ) - print(f"{EXTS[ext]} comparison {ipos + 1}", self.name) - + print(f"{EXTS[ext]} comparison {i + 1}", self.name) if not success: - break - + return False return True # otherwise it's a regression comparison files0, files1 = get_regression_files(self.workspace, extensions) extension = "hds" - ipos = 0 - success = True - for idx, (fpth0, fpth1) in enumerate(zip(files0, files1)): + for i, (fpth0, fpth1) in enumerate(zip(files0, files1)): outfile = os.path.splitext(os.path.basename(fpth0))[0] outfile = os.path.join( self.workspace, outfile + f".{extension}.cmp.out" @@ -411,26 +377,21 @@ def _compare_heads( files2=fpth1, verbose=self.verbose, ) - msg = ( - f"{EXTS[extension]} comparison {ipos + 1}" + print(( + f"{EXTS[extension]} comparison {i + 1}" + f"{self.name} ({os.path.basename(fpth0)})" - ) - ipos += 1 - print(msg) - + )) if not success: - break - - return success + return False + return True def _compare_concentrations(self, extensions="ucn") -> bool: if isinstance(extensions, str): extensions = [extensions] - success = True + files0, files1 = get_regression_files(self.workspace, extensions) extension = "ucn" - ipos = 0 - for fpth0, fpth1 in zip(files0, files1): + for i, (fpth0, fpth1) in enumerate(zip(files0, files1)): outfile = os.path.splitext(os.path.basename(fpth0))[0] outfile = os.path.join( self.workspace, outfile + f".{extension}.cmp.out" @@ -446,40 +407,34 @@ def _compare_concentrations(self, extensions="ucn") -> bool: files2=fpth1, verbose=self.verbose, ) - msg = ( - f"{EXTS[extension]} comparison {ipos + 1}" + print(( + f"{EXTS[extension]} comparison {i + 1}" + f"{self.name} ({os.path.basename(fpth0)})", - ) - ipos += 1 - print(msg) - + )) if not success: - break - - return success + return False + return True def _compare_budgets(self, extensions="cbc") -> bool: if isinstance(extensions, str): extensions = [extensions] - success = True files0, files1 = get_regression_files(self.workspace, extensions) extension = "cbc" - ipos = 0 - for fpth0, fpth1 in zip(files0, files1): + for i, (fpth0, fpth1) in enumerate(zip(files0, files1)): + print( + f"{EXTS[extension]} comparison {i + 1}", + f"{self.name} ({os.path.basename(fpth0)})", + ) success = self._compare_budget_files( - ipos, extension, fpth0, fpth1, ) - ipos += 1 - if not success: - break - - return success + return False + return True - def _compare_budget_files(self, ipos, extension, fpth0, fpth1) -> bool: + def _compare_budget_files(self, extension, fpth0, fpth1) -> bool: success = True if os.stat(fpth0).st_size * os.stat(fpth0).st_size == 0: return success, "" @@ -563,12 +518,7 @@ def _compare_budget_files(self, ipos, extension, fpth0, fpth1) -> bool: if self.verbose: print(msg) - print( - f"{EXTS[extension]} comparison {ipos + 1}", - f"{self.name} ({os.path.basename(fpth0)})", - ) fcmp.close() - return success # public @@ -744,7 +694,7 @@ def compare_output(self, compare="auto"): ), "concentration comparison failed" else: assert self._compare_heads( - compare=compare, cpth=cmp_path, extensions=hds_ext + cpth=cmp_path, extensions=hds_ext ), "head comparison failed" def run(self): @@ -817,13 +767,14 @@ def run(self): shutil.copytree(self.workspace, cmp_path) # run comparison model(s) with the comparison executable, key can be + # - mf6 + # - mf6_regression + # - libmf6 # - mf2005 # - mfnwt # - mfusg # - mflgr - # - libmf6 - # - mf6 - # - mf6_regression + # todo: don't hardcode workspace & assume agreement with test case # simulation workspace, store & access workspaces directly run_only = ( @@ -836,19 +787,11 @@ def run(self): ) assert self.run_sim_or_model( workspace, - self.targets.get( - self.compare.lower().replace(".cmp", ""), - self.targets.mf6, - ), + self.targets.get(self.compare, self.targets.mf6), ), f"Comparison model failed: {workspace}" # compare model results, if enabled if not run_only: - # if mf6 or mf6 regression test, get output files - if "mf6" in self.compare: - _, self.coutp = get_mf6_files( - self.workspace / "mfsim.nam", self.verbose - ) if self.verbose: print("Comparing model outputs") self.compare_output(self.compare) From 7695b4039eb1f9a68a9caff8cbfaf65f55b0b3b0 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 21 Dec 2023 22:15:12 -0500 Subject: [PATCH 2/2] fix _compare_budget_files() usage --- autotest/test_gwf_mf6io_app2_examples_distypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autotest/test_gwf_mf6io_app2_examples_distypes.py b/autotest/test_gwf_mf6io_app2_examples_distypes.py index 47875d324a7..3c0bbe5e928 100644 --- a/autotest/test_gwf_mf6io_app2_examples_distypes.py +++ b/autotest/test_gwf_mf6io_app2_examples_distypes.py @@ -551,7 +551,7 @@ def check_output(idx, test): fpth0 = ws / f"{sim_name}.{extension}" # fpth1 = ws / f"mf6/{get_dis_name(name)}.{extension}" fpth1 = ws / f"mf6/{sim_name}.{extension}" - test._compare_budget_files(0, extension, fpth0, fpth1) + test._compare_budget_files(extension, fpth0, fpth1) @pytest.mark.parametrize(