wip

autotest/test_dfn.py

@@ -1,65 +1,63 @@
 from pathlib import Path
 
+import pytest
+
 from modflow_devtools.dfn import Dfn, get_dfns
 from modflow_devtools.dfn2toml import convert
 from modflow_devtools.markers import requires_pkg
 
 PROJ_ROOT = Path(__file__).parents[1]
-DFN_PATH = PROJ_ROOT / "autotest" / "temp" / "dfn"
-TOML_PATH = DFN_PATH / "toml"
+DFN_DIR = PROJ_ROOT / "autotest" / "temp" / "dfn"
+TOML_DIR = DFN_DIR / "toml"
+VERSIONS = {1: DFN_DIR, 2: TOML_DIR}
+DFN_PATHS = list(DFN_DIR.glob("*.dfn"))
 MF6_OWNER = "MODFLOW-USGS"
 MF6_REPO = "modflow6"
 MF6_REF = "develop"
 
 
 def pytest_generate_tests(metafunc):
     if "dfn_name" in metafunc.fixturenames:
-        if not any(DFN_PATH.glob("*.dfn")):
-            get_dfns(MF6_OWNER, MF6_REPO, MF6_REF, DFN_PATH, verbose=True)
+        if not any(DFN_DIR.glob("*.dfn")):
+            get_dfns(MF6_OWNER, MF6_REPO, MF6_REF, DFN_DIR, verbose=True)
         dfn_names = [
             dfn.stem
-            for dfn in DFN_PATH.glob("*.dfn")
+            for dfn in DFN_DIR.glob("*.dfn")
             if dfn.stem not in ["common", "flopy"]
         ]
         metafunc.parametrize("dfn_name", dfn_names, ids=dfn_names)
 
     if "toml_name" in metafunc.fixturenames:
-        convert(DFN_PATH, TOML_PATH)
-        dfns = list(DFN_PATH.glob("*.dfn"))
+        convert(DFN_DIR, TOML_DIR)
         assert all(
-            (TOML_PATH / f"{dfn.stem}.toml").is_file()
-            for dfn in dfns
+            (TOML_DIR / f"{dfn.stem}.toml").is_file()
+            for dfn in DFN_PATHS
             if "common" not in dfn.stem
         )
-        toml_names = [toml.stem for toml in TOML_PATH.glob("*.toml")]
+        toml_names = [toml.stem for toml in TOML_DIR.glob("*.toml")]
         metafunc.parametrize("toml_name", toml_names, ids=toml_names)
 
 
 @requires_pkg("boltons")
 def test_load_v1(dfn_name):
     with (
-        (DFN_PATH / "common.dfn").open() as common_file,
-        (DFN_PATH / f"{dfn_name}.dfn").open() as dfn_file,
+        (DFN_DIR / "common.dfn").open() as common_file,
+        (DFN_DIR / f"{dfn_name}.dfn").open() as dfn_file,
     ):
         common, _ = Dfn._load_v1_flat(common_file)
         dfn = Dfn.load(dfn_file, name=dfn_name, common=common)
         assert any(dfn)
 
 
-@requires_pkg("boltons")
-def test_load_all_v1():
-    dfns = Dfn.load_all(DFN_PATH)
-    assert any(dfns)
-
-
 @requires_pkg("boltons")
 def test_load_v2(toml_name):
-    with (TOML_PATH / f"{toml_name}.toml").open(mode="rb") as toml_file:
+    with (TOML_DIR / f"{toml_name}.toml").open(mode="rb") as toml_file:
         toml = Dfn.load(toml_file, name=toml_name, version=2)
         assert any(toml)
 
 
 @requires_pkg("boltons")
-def test_load_all_v2():
-    toml = Dfn.load_all(TOML_PATH, version=2)
-    assert any(toml)
+@pytest.mark.parametrize("version", list(VERSIONS.keys()))
+def test_load_all(version):
+    dfns = Dfn.load_all(VERSIONS[version], version=version)
+    assert len(dfns) == len(DFN_PATHS) - 1  # ignore common.dfn

modflow_devtools/dfn.py

@@ -21,6 +21,7 @@
 
 import tomli
 from boltons.dictutils import OMD
+from boltons.iterutils import remap
 
 from modflow_devtools.download import download_and_unzip
 
@@ -48,6 +49,39 @@ def _try_parse_bool(value: Any) -> Any:
     return value
 
 
+def _var_attr_sort_key(item) -> int:
+    """
+    Sort key for input variables. The order is:
+    0. name
+    1. type
+    2. shape
+    3. default
+    4. reader
+    5. optional
+    6. longname
+    7. description
+    """
+
+    k, _ = item
+    if k == "name":
+        return 0
+    if k == "type":
+        return 1
+    if k == "shape":
+        return 2
+    if k == "default":
+        return 3
+    if k == "reader":
+        return 4
+    if k == "optional":
+        return 5
+    if k == "longname":
+        return 6
+    if k == "description":
+        return 7
+    return 8
+
+
 _MF6_SCALARS = {
     "keyword",
     "integer",
@@ -273,7 +307,7 @@ def _item() -> Var:
 
                 # implicit simple record (no children)
                 if all(t in _MF6_SCALARS for t in item_types):
-                    return Var(
+                    return dict(sorted(Var(
                         name=_name,
                         type="record",
                         block=block,
@@ -282,7 +316,7 @@ def _item() -> Var:
                             "is the list of", "is the record of"
                         ),
                         **var,
-                    )
+                    ).items(), key=_var_attr_sort_key))
 
                 # implicit complex record (has children)
                 fields = {
@@ -295,7 +329,7 @@ def _item() -> Var:
                 item_type = (
                     "union" if single and "keystring" in first["type"] else "record"
                 )
-                return Var(
+                return dict(sorted(Var(
                     name=first["name"] if single else _name,
                     type=item_type,
                     block=block,
@@ -304,7 +338,7 @@ def _item() -> Var:
                         "is the list of", f"is the {item_type} of"
                     ),
                     **var,
-                )
+                ).items(), key=_var_attr_sort_key))
 
             def _choices() -> Vars:
                 """Load a union's choices."""
@@ -330,14 +364,14 @@ def _fields() -> Vars:
                     fields[name] = v
                 return fields
 
-            var_ = Var(
+            var_ = dict(sorted(Var(
                 name=_name,
                 shape=shape,
                 block=block,
                 description=description,
                 default=default,
                 **var,
-            )
+            ).items(), key=_var_attr_sort_key))
 
             if _type.startswith("recarray"):
                 var_["item"] = _item()
@@ -362,7 +396,7 @@ def _fields() -> Vars:
 
             # if var is a foreign key, return subpkg var instead
             if ref:
-                return Var(
+                return dict(sorted(Var(
                     name=ref["param" if name == ("sim", "nam") else "val"],
                     type=_type,
                     shape=shape,
@@ -378,9 +412,9 @@ def _fields() -> Vars:
                     default=None,
                     subpackage=ref,
                     **var,
-                )
+                ).items(), key=_var_attr_sort_key))
 
-            return var_
+            return dict(sorted(var_.items(), key=_var_attr_sort_key))
 
         # load top-level variables. any nested
         # variables will be loaded recursively
@@ -396,8 +430,21 @@ def _fields() -> Vars:
             for block_name, block in groupby(vars_.values(), lambda v: v["block"])
         }
 
+        # remove block attribute
+        def remove_block(path, key, value):
+            if key == "block":
+                return False
+            return True
+        blocks = remap(blocks, visit=remove_block)
+
+        del_attrs = [
+            "in_record",
+            "tagged",
+            "preserve_case"
+        ]
+
         # collapse single-variable blocks
-        # def _is_list(block) -> bool:
+        # def _is_list-> bool:
         #     return next(iter(block.values()))["type"] == "list"
 
         # def _try_collapse(block_name, block) -> tuple:
@@ -507,7 +554,7 @@ def load(
         **kwargs,
     ) -> "Dfn":
         """
-        Load a component definition from a DFN file.
+        Load a component definition from a definition file.
         """
 
         if version == 1:
@@ -519,7 +566,6 @@ def load(
 
     @staticmethod
     def _load_all_v1(dfndir: PathLike) -> Dfns:
-        # find definition files
         paths: list[Path] = [
             p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"]
         ]
@@ -537,7 +583,7 @@ def _load_all_v1(dfndir: PathLike) -> Dfns:
         for path in paths:
             with path.open() as f:
                 dfn = Dfn.load(f, name=path.stem, common=common)
-                subpkg = dfn.get("subpackage", None)
+                subpkg = dfn.get("sub", None)
                 if subpkg:
                     refs[subpkg["key"]] = subpkg
 
@@ -552,12 +598,9 @@ def _load_all_v1(dfndir: PathLike) -> Dfns:
 
     @staticmethod
     def _load_all_v2(dfndir: PathLike) -> Dfns:
-        # find definition files
         paths: list[Path] = [
             p for p in dfndir.glob("*.toml") if p.stem not in ["common", "flopy"]
         ]
-
-        # load definitions
         dfns: Dfns = {}
         for path in paths:
             with path.open(mode="rb") as f: