add zn.Meta for metadata which is not tracked by DVC (#404)

* add `zn.Meta` for metadata which is not tracked by DVC

* Merge branch 'main' into 294-...

* add 'zntrack.meta.Text' ZnTrackOption

* fix merge conflicts

* improve test

tests/integration_tests/test_meta.py

@@ -0,0 +1,53 @@
+import pathlib
+import subprocess
+
+import pytest
+import yaml
+
+import zntrack
+
+
+class NodeWithMeta(zntrack.Node):
+    author = zntrack.meta.Text("Fabian")
+    title = zntrack.meta.Text("Test Node")
+
+
+def test_NodeWithMeta(proj_path):
+    NodeWithMeta().write_graph()
+
+    node_w_meta = NodeWithMeta.load()
+    assert node_w_meta.author == "Fabian"
+
+    dvc_yaml = yaml.safe_load(pathlib.Path("dvc.yaml").read_text())
+    assert dvc_yaml["stages"]["NodeWithMeta"]["meta"] == {
+        "author": "Fabian",
+        "title": "Test Node",
+    }
+
+
+class CombinedNodeWithMeta(zntrack.Node):
+    input: str = zntrack.zn.params("Hello ")
+    output: str = zntrack.zn.outs()
+    author: str = zntrack.meta.Text()
+
+    def run(self):
+        self.output = self.input + self.author
+
+
+def test_CombinedNodeWithMeta(proj_path):
+    with pytest.raises(TypeError):
+        # should raise an error because author is missing as kwarg
+        _ = CombinedNodeWithMeta()
+
+    CombinedNodeWithMeta(author="World").write_graph(run=True)
+    assert CombinedNodeWithMeta.load().output == "Hello World"
+    assert CombinedNodeWithMeta.load().author == "World"
+
+    CombinedNodeWithMeta(author="there").write_graph(run=True)
+    # changing the 'meta.Text' should not trigger running the model again
+    assert CombinedNodeWithMeta.load().output == "Hello World"
+    assert CombinedNodeWithMeta.load().author == "there"
+
+    subprocess.check_call(["dvc", "repro", "-f"])
+    # Forcing rerun should use the updated meta keyword.
+    assert CombinedNodeWithMeta.load().output == "Hello there"

tests/unit_tests/utils/test_file_io.py

@@ -243,3 +243,38 @@ def pathlib_open(*args, **kwargs):
         open_mock().write.assert_called_once_with(yaml.safe_dump(dvc_dict, indent=4))
     else:
         assert not open_mock().called
+
+
+@pytest.mark.parametrize("data", ({"author": "Fabian"}, None))
+def test_update_meta(data):
+    dvc_dict = {"stages": {"MyNode": {"cmd": "run", "meta": {"a": "b"}}}}
+
+    open_mock = mock_open(read_data=yaml.safe_dump(dvc_dict))
+
+    def pathlib_open(*args, **kwargs):
+        return open_mock(*args, **kwargs)
+
+    file = pathlib.Path("dvc.yaml")
+
+    with patch.object(pathlib.Path, "open", pathlib_open):
+        file_io.update_meta(file=file, node_name="MyNode", data=data)
+
+    if data is not None:
+        dvc_dict["stages"]["MyNode"]["meta"].update(data)
+        open_mock().write.assert_called_once_with(yaml.safe_dump(dvc_dict, indent=4))
+    else:
+        assert not open_mock().called
+
+
+def test_update_meta_existing():
+    dvc_dict = {"stages": {"MyNode": {"meta": "not a dict"}}}
+    open_mock = mock_open(read_data=yaml.safe_dump(dvc_dict))
+
+    def pathlib_open(*args, **kwargs):
+        return open_mock(*args, **kwargs)
+
+    file = pathlib.Path("dvc.yaml")
+
+    with patch.object(pathlib.Path, "open", pathlib_open):
+        with pytest.raises(ValueError):
+            file_io.update_meta(file=file, node_name="MyNode", data={"a": "b"})

zntrack/core/base.py

@@ -11,7 +11,7 @@
 import zninit
 import znjson
 
-from zntrack import dvc, utils, zn
+from zntrack import dvc, meta, utils, zn
 from zntrack.core.dvcgraph import (
     DVCRunOptions,
     ZnTrackInfo,
@@ -195,7 +195,7 @@ class Node(NodeBase, metaclass=LoadViaGetItem):
     """
 
     init_subclass_basecls = NodeBase
-    init_descriptors = [zn.params, zn.deps, zn.Method, zn.Nodes] + dvc.options
+    init_descriptors = [zn.params, zn.deps, zn.Method, zn.Nodes, meta.Text] + dvc.options
 
     @utils.deprecated(
         reason=(
@@ -283,7 +283,7 @@ def save(self, results: bool = False, hash_only: bool = False):
                     # only save results
                     option.save(instance=self)
             else:
-                if option.zn_type not in utils.VALUE_DVC_TRACKED:
+                if option.zn_type not in utils.VALUE_DVC_TRACKED + utils.GIT_TRACKED:
                     # save all dvc.<options>
                     option.save(instance=self)
                 else:
@@ -591,6 +591,10 @@ def write_graph(
 
         run_post_dvc_cmd(descriptor_list=self._descriptor_list, instance=self)
 
+        for option in self._descriptor_list:
+            if option.zn_type in utils.GIT_TRACKED:
+                option.save(instance=self)
+
         if write_desc:
             utils.file_io.update_desc(
                 file=utils.Files.dvc, node_name=self.node_name, desc=self.__doc__

zntrack/meta/__init__.py

@@ -0,0 +1,51 @@
+"""ZnTrack Node meta data __init__
+
+Collection of Nodes that allow the storage of data inside a Node that is neither a true
+parameter but also not an output. This can be e.g. information about the user, some
+description but also e.g. different paths to binaries that should be used.
+In principle, it is advisable to add the binary as a direct dependency but often
+this is not feasible and can be circumvented by using 'meta'.
+"""
+import pathlib
+
+from zntrack import utils
+from zntrack.core.zntrackoption import ZnTrackOption
+
+
+class Text(ZnTrackOption):
+    """ZnTrack Text based meta descriptor
+
+    This ZnTrackOption allows the storage of plain text data in the 'dvc.yaml' meta key.
+    """
+
+    zn_type = utils.ZnTypes.META
+    file = utils.Files.dvc
+    dvc_option = utils.DVCOptions.PARAMS
+
+    def get_filename(self, instance) -> pathlib.Path:
+        return
+
+    def mkdir(self, instance):
+        return
+
+    def get_data_from_files(self, instance):
+        try:
+            file_content = utils.file_io.read_file(self.file)
+        except FileNotFoundError as err:
+            raise self._get_loading_errors(instance) from err
+
+        try:
+            values = file_content["stages"][instance.node_name]["meta"][self.name]
+        except KeyError as err:
+            raise self._get_loading_errors(instance) from err
+        return values
+
+    def save(self, instance):
+        if instance.__dict__.get(self.name) is utils.LazyOption:
+            # do not save anything if __get__/__set__ was never used
+            return
+        utils.file_io.update_meta(
+            file=utils.Files.dvc,
+            node_name=instance.node_name,
+            data={self.name: getattr(instance, self.name)},
+        )

zntrack/utils/__init__.py

@@ -5,6 +5,7 @@
 from zntrack.utils.nwd import nwd
 from zntrack.utils.structs import (
     FILE_DVC_TRACKED,
+    GIT_TRACKED,
     VALUE_DVC_TRACKED,
     DVCOptions,
     LazyOption,
@@ -45,4 +46,5 @@
     "LazyOption",
     "helpers",
     "nwd",
+    "GIT_TRACKED",
 ]

zntrack/utils/file_io.py

@@ -139,3 +139,18 @@ def update_desc(file: pathlib.Path, node_name: str, desc: str):
         file_content = read_file(file)
         file_content["stages"][node_name]["desc"] = desc
         write_file(file, value=file_content)
+
+
+def update_meta(file: pathlib.Path, node_name: str, data: dict):
+    """Update the file (dvc.yaml) given the Node for 'meta' key with the data"""
+    if data is not None:
+        file_content = read_file(file)
+        meta_data = file_content["stages"][node_name].get("meta", {})
+        if not isinstance(meta_data, dict):
+            raise ValueError(
+                "The 'meta' key in the 'dvc.yaml' is not empty or was otherwise modified."
+                " To use 'zntrack.meta' it is not possible to use that field otherwise."
+            )
+        meta_data.update(data)
+        file_content["stages"][node_name]["meta"] = meta_data
+        write_file(file, value=file_content)

zntrack/utils/structs.py

@@ -27,17 +27,22 @@ class ZnTypes(enum.Enum):
     RESULTS = enum.auto()
     PLOTS = enum.auto()
     HASH = enum.auto()
+    META = enum.auto()
 
 
-FILE_DVC_TRACKED = (ZnTypes.DVC,)
+FILE_DVC_TRACKED = [ZnTypes.DVC]
 # if the getattr(instance, self.name) is an affected file,
 # e.g. the dvc.<outs> is a file / list of files
-VALUE_DVC_TRACKED = (
+
+VALUE_DVC_TRACKED = [
     ZnTypes.RESULTS,
     ZnTypes.METADATA,
     ZnTypes.PLOTS,
     ZnTypes.HASH,
-)
+]
+
+# The data is not tracked / used via any DVC command
+GIT_TRACKED = [ZnTypes.META]
 
 
 # if the internal file,