ENH: Handle None types (#742)

* added support for None to be implicitly interpreted as NoneType in type-checking/coercing

* fix up type checking of none-types

* write file in dir input change

* implemented UnionType type-parsing support

* added register_serializer for types.UnionType

* fixed error in runtime error

* reverted test change

* fixed exception pattern matching to cover head and latest release of fileformats

* adopted ghislain's suggestion

Author: tclose

pydra/engine/specs.py

@@ -991,8 +991,8 @@ def get_value(
                 f"named '{node.checksum}' in any of the cache locations.\n"
                 + "\n".join(str(p) for p in set(node.cache_locations))
                 + f"\n\nThis is likely due to hash changes in '{self.name}' node inputs. "
-                f"Current values and hashes: {self.inputs}, "
-                f"{self.inputs._hashes}\n\n"
+                f"Current values and hashes: {node.inputs}, "
+                f"{node.inputs._hashes}\n\n"
                 "Set loglevel to 'debug' in order to track hash changes "
                 "throughout the execution of the workflow.\n\n "
                 "These issues may have been caused by `bytes_repr()` methods "

pydra/utils/hash.py

@@ -1,9 +1,11 @@
 """Generic object hashing dispatch"""
 
+import sys
 import os
 import struct
 from datetime import datetime
 import typing as ty
+import types
 from pathlib import Path
 from collections.abc import Mapping
 from functools import singledispatch
@@ -467,6 +469,10 @@ def type_name(tp):
     yield b")"
 
 
+if sys.version_info >= (3, 10):
+    register_serializer(types.UnionType)(bytes_repr_type)
+
+
 @register_serializer(FileSet)
 def bytes_repr_fileset(
     fileset: FileSet, cache: Cache

pydra/utils/tests/test_hash.py

@@ -1,5 +1,6 @@
 import re
 import os
+import sys
 from hashlib import blake2b
 from pathlib import Path
 import time
@@ -200,6 +201,14 @@ def test_bytes_special_form1():
     assert obj_repr == b"type:(typing.Union[type:(builtins.int)type:(builtins.float)])"
 
 
+@pytest.mark.skipif(condition=sys.version_info < (3, 10), reason="requires python3.10")
+def test_bytes_special_form1a():
+    obj_repr = join_bytes_repr(int | float)
+    assert (
+        obj_repr == b"type:(types.UnionType[type:(builtins.int)type:(builtins.float)])"
+    )
+
+
 def test_bytes_special_form2():
     obj_repr = join_bytes_repr(ty.Any)
     assert re.match(rb"type:\(typing.Any\)", obj_repr)
@@ -212,6 +221,15 @@ def test_bytes_special_form3():
     )
 
 
+@pytest.mark.skipif(condition=sys.version_info < (3, 10), reason="requires python3.10")
+def test_bytes_special_form3a():
+    obj_repr = join_bytes_repr(Path | None)
+    assert (
+        obj_repr
+        == b"type:(types.UnionType[type:(pathlib.Path)type:(builtins.NoneType)])"
+    )
+
+
 def test_bytes_special_form4():
     obj_repr = join_bytes_repr(ty.Type[Path])
     assert obj_repr == b"type:(builtins.type[type:(pathlib.Path)])"

pydra/utils/tests/test_typing.py

@@ -120,13 +120,26 @@ def test_type_check_basic15():
     TypeParser(ty.Union[Path, File, float])(lz(int))
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_check_basic15a():
+    TypeParser(Path | File | float)(lz(int))
+
+
 def test_type_check_basic16():
     with pytest.raises(
         TypeError, match="Cannot coerce <class 'float'> to any of the union types"
     ):
         TypeParser(ty.Union[Path, File, bool, int])(lz(float))
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_check_basic16a():
+    with pytest.raises(
+        TypeError, match="Cannot coerce <class 'float'> to any of the union types"
+    ):
+        TypeParser(Path | File | bool | int)(lz(float))
+
+
 def test_type_check_basic17():
     TypeParser(ty.Sequence)(lz(ty.Tuple[int, ...]))
 
@@ -194,6 +207,12 @@ def test_type_check_fail2():
         TypeParser(ty.Union[Path, File])(lz(int))
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_check_fail2a():
+    with pytest.raises(TypeError, match="to any of the union types"):
+        TypeParser(Path | File)(lz(int))
+
+
 def test_type_check_fail3():
     with pytest.raises(TypeError, match="doesn't match any of the explicit inclusion"):
         TypeParser(ty.Sequence, coercible=[(ty.Sequence, ty.Sequence)])(
@@ -312,13 +331,32 @@ def test_type_coercion_basic12():
     assert TypeParser(ty.Union[Path, File, int], coercible=[(ty.Any, ty.Any)])(1.0) == 1
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_coercion_basic12a():
+    with pytest.raises(TypeError, match="explicitly excluded"):
+        TypeParser(
+            list,
+            coercible=[(ty.Sequence, ty.Sequence)],
+            not_coercible=[(str, ty.Sequence)],
+        )("a-string")
+
+    assert TypeParser(Path | File | int, coercible=[(ty.Any, ty.Any)])(1.0) == 1
+
+
 def test_type_coercion_basic13():
     assert (
         TypeParser(ty.Union[Path, File, bool, int], coercible=[(ty.Any, ty.Any)])(1.0)
         is True
     )
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_coercion_basic13a():
+    assert (
+        TypeParser(Path | File | bool | int, coercible=[(ty.Any, ty.Any)])(1.0) is True
+    )
+
+
 def test_type_coercion_basic14():
     assert TypeParser(ty.Sequence, coercible=[(ty.Any, ty.Any)])((1, 2, 3)) == (
         1,
@@ -404,6 +442,12 @@ def test_type_coercion_fail2():
         TypeParser(ty.Union[Path, File], coercible=[(ty.Any, ty.Any)])(1)
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_coercion_fail2a():
+    with pytest.raises(TypeError, match="to any of the union types"):
+        TypeParser(Path | File, coercible=[(ty.Any, ty.Any)])(1)
+
+
 def test_type_coercion_fail3():
     with pytest.raises(TypeError, match="doesn't match any of the explicit inclusion"):
         TypeParser(ty.Sequence, coercible=[(ty.Sequence, ty.Sequence)])(
@@ -446,7 +490,7 @@ def f(x: ty.List[File], y: ty.Dict[str, ty.List[File]]):
     TypeParser(ty.List[str])(task.lzout.a)  # pylint: disable=no-member
     with pytest.raises(
         TypeError,
-        match="Cannot coerce <class 'fileformats.generic.File'> into <class 'int'>",
+        match="Cannot coerce <class 'fileformats\.generic.*\.File'> into <class 'int'>",
     ):
         TypeParser(ty.List[int])(task.lzout.a)  # pylint: disable=no-member
 
@@ -469,6 +513,27 @@ def test_matches_type_union():
     assert not TypeParser.matches_type(ty.Union[int, bool, str], ty.Union[int, bool])
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_matches_type_union_a():
+    assert TypeParser.matches_type(int | bool | str, int | bool | str)
+    assert TypeParser.matches_type(int | bool, int | bool | str)
+    assert not TypeParser.matches_type(int | bool | str, int | bool)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_matches_type_union_b():
+    assert TypeParser.matches_type(int | bool | str, ty.Union[int, bool, str])
+    assert TypeParser.matches_type(int | bool, ty.Union[int, bool, str])
+    assert not TypeParser.matches_type(int | bool | str, ty.Union[int, bool])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_matches_type_union_c():
+    assert TypeParser.matches_type(ty.Union[int, bool, str], int | bool | str)
+    assert TypeParser.matches_type(ty.Union[int, bool], int | bool | str)
+    assert not TypeParser.matches_type(ty.Union[int, bool, str], int | bool)
+
+
 def test_matches_type_dict():
     COERCIBLE = [(str, Path), (Path, str), (int, float)]
 
@@ -713,18 +778,61 @@ def test_union_is_subclass1():
     assert TypeParser.is_subclass(ty.Union[Json, Yaml], ty.Union[Json, Yaml, Xml])
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass1a():
+    assert TypeParser.is_subclass(Json | Yaml, Json | Yaml | Xml)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass1b():
+    assert TypeParser.is_subclass(Json | Yaml, ty.Union[Json, Yaml, Xml])
+
+
+## Up to here!
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass1c():
+    assert TypeParser.is_subclass(ty.Union[Json, Yaml], Json | Yaml | Xml)
+
+
 def test_union_is_subclass2():
     assert not TypeParser.is_subclass(ty.Union[Json, Yaml, Xml], ty.Union[Json, Yaml])
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass2a():
+    assert not TypeParser.is_subclass(Json | Yaml | Xml, Json | Yaml)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass2b():
+    assert not TypeParser.is_subclass(ty.Union[Json, Yaml, Xml], Json | Yaml)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass2c():
+    assert not TypeParser.is_subclass(Json | Yaml | Xml, ty.Union[Json, Yaml])
+
+
 def test_union_is_subclass3():
     assert TypeParser.is_subclass(Json, ty.Union[Json, Yaml])
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass3a():
+    assert TypeParser.is_subclass(Json, Json | Yaml)
+
+
 def test_union_is_subclass4():
     assert not TypeParser.is_subclass(ty.Union[Json, Yaml], Json)
 
 
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_union_is_subclass4a():
+    assert not TypeParser.is_subclass(Json | Yaml, Json)
+
+
 def test_generic_is_subclass1():
     assert TypeParser.is_subclass(ty.List[int], list)
 
@@ -737,6 +845,56 @@ def test_generic_is_subclass3():
     assert not TypeParser.is_subclass(ty.List[float], ty.List[int])
 
 
+def test_none_is_subclass1():
+    assert TypeParser.is_subclass(None, ty.Union[int, None])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_none_is_subclass1a():
+    assert TypeParser.is_subclass(None, int | None)
+
+
+def test_none_is_subclass2():
+    assert not TypeParser.is_subclass(None, ty.Union[int, float])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_none_is_subclass2a():
+    assert not TypeParser.is_subclass(None, int | float)
+
+
+def test_none_is_subclass3():
+    assert TypeParser.is_subclass(ty.Tuple[int, None], ty.Tuple[int, None])
+
+
+def test_none_is_subclass4():
+    assert TypeParser.is_subclass(None, None)
+
+
+def test_none_is_subclass5():
+    assert not TypeParser.is_subclass(None, int)
+
+
+def test_none_is_subclass6():
+    assert not TypeParser.is_subclass(int, None)
+
+
+def test_none_is_subclass7():
+    assert TypeParser.is_subclass(None, type(None))
+
+
+def test_none_is_subclass8():
+    assert TypeParser.is_subclass(type(None), None)
+
+
+def test_none_is_subclass9():
+    assert TypeParser.is_subclass(type(None), type(None))
+
+
+def test_none_is_subclass10():
+    assert TypeParser.is_subclass(type(None), type(None))
+
+
 @pytest.mark.skipif(
     sys.version_info < (3, 9), reason="Cannot subscript tuple in < Py3.9"
 )
@@ -780,3 +938,46 @@ def test_type_is_instance3():
 
 def test_type_is_instance4():
     assert TypeParser.is_instance(Json, type)
+
+
+def test_type_is_instance5():
+    assert TypeParser.is_instance(None, None)
+
+
+def test_type_is_instance6():
+    assert TypeParser.is_instance(None, type(None))
+
+
+def test_type_is_instance7():
+    assert not TypeParser.is_instance(None, int)
+
+
+def test_type_is_instance8():
+    assert not TypeParser.is_instance(1, None)
+
+
+def test_type_is_instance9():
+    assert TypeParser.is_instance(None, ty.Union[int, None])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_is_instance9a():
+    assert TypeParser.is_instance(None, int | None)
+
+
+def test_type_is_instance10():
+    assert TypeParser.is_instance(1, ty.Union[int, None])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_is_instance10a():
+    assert TypeParser.is_instance(1, int | None)
+
+
+def test_type_is_instance11():
+    assert not TypeParser.is_instance(None, ty.Union[int, str])
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="No UnionType < Py3.10")
+def test_type_is_instance11a():
+    assert not TypeParser.is_instance(None, int | str)