From 376ae56396ba35471201ef298291677fbc7e79f2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Oct 2024 10:17:56 +0200 Subject: [PATCH 01/15] Fix get_type_hints with None default interaction --- CHANGELOG.md | 4 +++ src/test_typing_extensions.py | 31 +++++++++++++++++ src/typing_extensions.py | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index db6719c6..81b4b953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ subscripted objects) had wrong parameters if they were directly subscripted with an `Unpack` object. Patch by [Daraan](https://github.com/Daraan). +- Fix backport of `get_type_hints` to reflect Python 3.11+ behavior which does not add + `Union[..., NoneType]` to annotations that have a `None` default value anymore. + This fixes wrapping of `Annotated` in an unwanted `Optional` in such cases. + Patch by [Daraan](https://github.com/Daraan). # Release 4.12.2 (June 7, 2024) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 8c2726f8..13dd2b7d 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -4993,6 +4993,37 @@ def test_nested_annotated_with_unhashable_metadata(self): self.assertEqual(X.__origin__, List[Annotated[str, {"unhashable_metadata"}]]) self.assertEqual(X.__metadata__, ("metadata",)) + def test_get_type_hints(self): + annotation = Annotated[Union[int, None], "data"] + optional_annotation = Optional[annotation] + + def wanted_optional(bar: optional_annotation): ... + def wanted_optional_default(bar: optional_annotation = None): ... + def wanted_optional_ref(bar: 'Optional[Annotated[Union[int, None], "data"]]'): ... + + def no_optional(bar: annotation): ... + def no_optional_default(bar: annotation = None): ... + def no_optional_defaultT(bar: Union[annotation, T] = None): ... + def no_optional_defaultT_ref(bar: "Union[annotation, T]" = None): ... + + for func in(wanted_optional, wanted_optional_default, wanted_optional_ref): + self.assertEqual( + get_type_hints(func, include_extras=True), + {"bar": optional_annotation} + ) + + for func in (no_optional, no_optional_default): + self.assertEqual( + get_type_hints(func, include_extras=True), + {"bar": annotation} + ) + + for func in (no_optional_defaultT, no_optional_defaultT_ref): + self.assertEqual( + get_type_hints(func, globals(), locals(), include_extras=True), + {"bar": Union[annotation, T]} + ) + class GetTypeHintsTests(BaseTestCase): def test_get_type_hints(self): diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 5bf4f2dc..a5d2a4b0 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1236,10 +1236,75 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): ) else: # 3.8 hint = typing.get_type_hints(obj, globalns=globalns, localns=localns) + if sys.version_info < (3, 11) and hint: + hint = _clean_optional(obj, hint, globalns, localns) if include_extras: return hint return {k: _strip_extras(t) for k, t in hint.items()} + _NoneType = type(None) + + def _could_be_inserted_optional(t): + """detects Union[..., None] pattern""" + # 3.8+ compatible checking before _UnionGenericAlias + if not hasattr(t, "__origin__") or t.__origin__ is not Union: + return False + # Assume if last argument is not None they are user defined + if t.__args__[-1] is not _NoneType: + return False + return True + + # < 3.11 + def _clean_optional(obj, hints, globalns=None, localns=None): + # reverts injected Union[..., None] cases from typing.get_type_hints + # when a None default value is used. + # see https://github.com/python/typing_extensions/issues/310 + original_hints = getattr(obj, '__annotations__', None) + defaults = typing._get_defaults(obj) + for name, value in hints.items(): + # Not a Union[..., None] or replacement conditions not fullfilled + if (not _could_be_inserted_optional(value) + or name not in defaults + or defaults[name] is not None + ): + continue + original_value = original_hints[name] + if original_value is None: + original_value = _NoneType + # Forward reference + if isinstance(original_value, str): + if globalns is None: + if isinstance(obj, _types.ModuleType): + globalns = obj.__dict__ + else: + nsobj = obj + # Find globalns for the unwrapped object. + while hasattr(nsobj, '__wrapped__'): + nsobj = nsobj.__wrapped__ + globalns = getattr(nsobj, '__globals__', {}) + if localns is None: + localns = globalns + elif localns is None: + localns = globalns + if sys.version_info < (3, 9): + ref = ForwardRef(original_value) + else: + ref = ForwardRef( + original_value, + is_argument=not isinstance(obj, _types.ModuleType) + ) + original_value = typing._eval_type(ref, globalns, localns) + # Values was not modified or original is already Optional + if original_value == value or _could_be_inserted_optional(original_value): + continue + # NoneType was added to value + if len(value.__args__) == 2: + hints[name] = value.__args__[0] # not a Union + else: + hints[name] = Union[value.__args__[:-1]] # still a Union + + return hints + # Python 3.9+ has PEP 593 (Annotated) if hasattr(typing, 'Annotated'): From 24c5602e8893dcbdf9be91d8f1eef0884cf414f2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Oct 2024 11:29:51 +0200 Subject: [PATCH 02/15] moved section --- src/test_typing_extensions.py | 61 ++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 13dd2b7d..47a4562a 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1645,6 +1645,37 @@ def test_final_forward_ref(self): self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) self.assertNotEqual(gth(Loop, globals())['attr'], Final) + def test_annotation_and_optional_default(self): + annotation = Annotated[Union[int, None], "data"] + optional_annotation = Optional[annotation] + + def wanted_optional(bar: optional_annotation): ... + def wanted_optional_default(bar: optional_annotation = None): ... + def wanted_optional_ref(bar: 'Optional[Annotated[Union[int, None], "data"]]'): ... + + def no_optional(bar: annotation): ... + def no_optional_default(bar: annotation = None): ... + def no_optional_defaultT(bar: Union[annotation, T] = None): ... + def no_optional_defaultT_ref(bar: "Union[annotation, T]" = None): ... + + for func in(wanted_optional, wanted_optional_default, wanted_optional_ref): + self.assertEqual( + get_type_hints(func, include_extras=True), + {"bar": optional_annotation} + ) + + for func in (no_optional, no_optional_default): + self.assertEqual( + get_type_hints(func, include_extras=True), + {"bar": annotation} + ) + + for func in (no_optional_defaultT, no_optional_defaultT_ref): + self.assertEqual( + get_type_hints(func, globals(), locals(), include_extras=True), + {"bar": Union[annotation, T]} + ) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -4993,36 +5024,6 @@ def test_nested_annotated_with_unhashable_metadata(self): self.assertEqual(X.__origin__, List[Annotated[str, {"unhashable_metadata"}]]) self.assertEqual(X.__metadata__, ("metadata",)) - def test_get_type_hints(self): - annotation = Annotated[Union[int, None], "data"] - optional_annotation = Optional[annotation] - - def wanted_optional(bar: optional_annotation): ... - def wanted_optional_default(bar: optional_annotation = None): ... - def wanted_optional_ref(bar: 'Optional[Annotated[Union[int, None], "data"]]'): ... - - def no_optional(bar: annotation): ... - def no_optional_default(bar: annotation = None): ... - def no_optional_defaultT(bar: Union[annotation, T] = None): ... - def no_optional_defaultT_ref(bar: "Union[annotation, T]" = None): ... - - for func in(wanted_optional, wanted_optional_default, wanted_optional_ref): - self.assertEqual( - get_type_hints(func, include_extras=True), - {"bar": optional_annotation} - ) - - for func in (no_optional, no_optional_default): - self.assertEqual( - get_type_hints(func, include_extras=True), - {"bar": annotation} - ) - - for func in (no_optional_defaultT, no_optional_defaultT_ref): - self.assertEqual( - get_type_hints(func, globals(), locals(), include_extras=True), - {"bar": Union[annotation, T]} - ) class GetTypeHintsTests(BaseTestCase): From 7f000b107a91662a3319f1f40ba8af2f248f84fd Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Oct 2024 11:32:13 +0200 Subject: [PATCH 03/15] removed empty line --- src/test_typing_extensions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 47a4562a..ace56caf 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -5025,7 +5025,6 @@ def test_nested_annotated_with_unhashable_metadata(self): self.assertEqual(X.__metadata__, ("metadata",)) - class GetTypeHintsTests(BaseTestCase): def test_get_type_hints(self): def foobar(x: List['X']): ... From 8cc4464f66a5b3dc7522974b9880eb12b21471d1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Oct 2024 18:34:33 +0200 Subject: [PATCH 04/15] Restructured tests and applied suggested changes --- src/test_typing_extensions.py | 65 +++++++++++++++++++++-------------- src/typing_extensions.py | 37 ++++++++++---------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index ace56caf..80df7b2d 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1649,32 +1649,47 @@ def test_annotation_and_optional_default(self): annotation = Annotated[Union[int, None], "data"] optional_annotation = Optional[annotation] - def wanted_optional(bar: optional_annotation): ... - def wanted_optional_default(bar: optional_annotation = None): ... - def wanted_optional_ref(bar: 'Optional[Annotated[Union[int, None], "data"]]'): ... - - def no_optional(bar: annotation): ... - def no_optional_default(bar: annotation = None): ... - def no_optional_defaultT(bar: Union[annotation, T] = None): ... - def no_optional_defaultT_ref(bar: "Union[annotation, T]" = None): ... - - for func in(wanted_optional, wanted_optional_default, wanted_optional_ref): - self.assertEqual( - get_type_hints(func, include_extras=True), - {"bar": optional_annotation} - ) - - for func in (no_optional, no_optional_default): - self.assertEqual( - get_type_hints(func, include_extras=True), - {"bar": annotation} - ) + cases = { + ((), False): {}, + ((), True): {}, + (int, False): {"x": int}, + (int, True): {"x": int}, + (Optional[int], False): {"x": Optional[int]}, + (Optional[int], True): {"x": Optional[int]}, + (optional_annotation, False): {"x": optional_annotation}, + (optional_annotation, True): {"x": optional_annotation}, + (str(optional_annotation), True): {"x": optional_annotation}, + (annotation, False): {"x": annotation}, + (annotation, True): {"x": annotation}, + (Union[annotation, T], False): {"x": Union[annotation, T]}, + (Union[annotation, T], True): {"x": Union[annotation, T]}, + (Union[str, None, "str"], False): {"x": Optional[str]}, + (Union[str, None, "str"], True): {"x": Optional[str]}, + (Union[str, "str"], False): { + "x": str + if sys.version_info >= (3, 9) + # _eval_type does not resolve correctly to str in 3.8 + else typing._eval_type(Union[str, "str"], None, None), + }, + (Union[str, "str"], True): {"x": str}, + (List["str"], False): {"x": List[str]}, + (List["str"], True): {"x": List[str]}, + (Optional[List[str]], False): {"x": Optional[List[str]]}, + (Optional[List[str]], True): {"x": Optional[List[str]]}, + } - for func in (no_optional_defaultT, no_optional_defaultT_ref): - self.assertEqual( - get_type_hints(func, globals(), locals(), include_extras=True), - {"bar": Union[annotation, T]} - ) + for (annot, none_default), expected in cases.items(): + with self.subTest(annotation=annot, none_default=none_default, expected_type_hints=expected): + if annot == (): + if none_default: + def func(x = None): pass + else: + def func(x): pass + elif none_default: + def func(x: annot = None): pass + else: + def func(x: annot): pass + self.assertEqual(get_type_hints(func, include_extras=True), expected) class GetUtilitiesTestCase(TestCase): diff --git a/src/typing_extensions.py b/src/typing_extensions.py index a5d2a4b0..925f9e52 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1236,8 +1236,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): ) else: # 3.8 hint = typing.get_type_hints(obj, globalns=globalns, localns=localns) - if sys.version_info < (3, 11) and hint: - hint = _clean_optional(obj, hint, globalns, localns) + if sys.version_info < (3, 11): + _clean_optional(obj, hint, globalns, localns) if include_extras: return hint return {k: _strip_extras(t) for k, t in hint.items()} @@ -1247,7 +1247,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): def _could_be_inserted_optional(t): """detects Union[..., None] pattern""" # 3.8+ compatible checking before _UnionGenericAlias - if not hasattr(t, "__origin__") or t.__origin__ is not Union: + if get_origin(t) is not Union: return False # Assume if last argument is not None they are user defined if t.__args__[-1] is not _NoneType: @@ -1259,8 +1259,12 @@ def _clean_optional(obj, hints, globalns=None, localns=None): # reverts injected Union[..., None] cases from typing.get_type_hints # when a None default value is used. # see https://github.com/python/typing_extensions/issues/310 - original_hints = getattr(obj, '__annotations__', None) + if not hints or isinstance(obj, type): + return defaults = typing._get_defaults(obj) + if not defaults: + return + original_hints = obj.__annotations__ for name, value in hints.items(): # Not a Union[..., None] or replacement conditions not fullfilled if (not _could_be_inserted_optional(value) @@ -1269,7 +1273,7 @@ def _clean_optional(obj, hints, globalns=None, localns=None): ): continue original_value = original_hints[name] - if original_value is None: + if original_value is None: # should not happen original_value = _NoneType # Forward reference if isinstance(original_value, str): @@ -1287,24 +1291,19 @@ def _clean_optional(obj, hints, globalns=None, localns=None): elif localns is None: localns = globalns if sys.version_info < (3, 9): - ref = ForwardRef(original_value) + original_value = ForwardRef(original_value) else: - ref = ForwardRef( + original_value = ForwardRef( original_value, is_argument=not isinstance(obj, _types.ModuleType) ) - original_value = typing._eval_type(ref, globalns, localns) - # Values was not modified or original is already Optional - if original_value == value or _could_be_inserted_optional(original_value): - continue - # NoneType was added to value - if len(value.__args__) == 2: - hints[name] = value.__args__[0] # not a Union - else: - hints[name] = Union[value.__args__[:-1]] # still a Union - - return hints - + original_evaluated = typing._eval_type(original_value, globalns, localns) + if sys.version_info < (3, 9) and get_origin(original_evaluated) is Union: + # Union[str, None, "str"] is not reduced to Union[str, None] + original_evaluated = Union[original_evaluated.__args__] + # Compare if values differ + if original_evaluated != value: + hints[name] = original_evaluated # Python 3.9+ has PEP 593 (Annotated) if hasattr(typing, 'Annotated'): From 881ffeeafaabb9c283674e1038564b1745d3f3ee Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Oct 2024 18:42:39 +0200 Subject: [PATCH 05/15] +1 test and formatting --- src/test_typing_extensions.py | 44 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 80df7b2d..f21fdec6 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1650,39 +1650,43 @@ def test_annotation_and_optional_default(self): optional_annotation = Optional[annotation] cases = { + # (annotation, none_default) : expected_type_hints ((), False): {}, ((), True): {}, - (int, False): {"x": int}, - (int, True): {"x": int}, - (Optional[int], False): {"x": Optional[int]}, - (Optional[int], True): {"x": Optional[int]}, - (optional_annotation, False): {"x": optional_annotation}, - (optional_annotation, True): {"x": optional_annotation}, - (str(optional_annotation), True): {"x": optional_annotation}, - (annotation, False): {"x": annotation}, - (annotation, True): {"x": annotation}, - (Union[annotation, T], False): {"x": Union[annotation, T]}, - (Union[annotation, T], True): {"x": Union[annotation, T]}, - (Union[str, None, "str"], False): {"x": Optional[str]}, - (Union[str, None, "str"], True): {"x": Optional[str]}, + (int, False): {'x': int}, + (int, True): {'x': int}, + (Optional[int], False): {'x': Optional[int]}, + (Optional[int], True): {'x': Optional[int]}, + (optional_annotation, False): {'x': optional_annotation}, + (optional_annotation, True): {'x': optional_annotation}, + (str(optional_annotation), True): {'x': optional_annotation}, + (annotation, False): {'x': annotation}, + (annotation, True): {'x': annotation}, + (Union[annotation, T], False): {'x': Union[annotation, T]}, + (Union[annotation, T], True): {'x': Union[annotation, T]}, + ("Union[Annotated[Union[int, None], 'data'], T]", True): { + 'x': Union[annotation, T] + }, + (Union[str, None, "str"], False): {'x': Optional[str]}, + (Union[str, None, "str"], True): {'x': Optional[str]}, (Union[str, "str"], False): { - "x": str + 'x': str if sys.version_info >= (3, 9) # _eval_type does not resolve correctly to str in 3.8 else typing._eval_type(Union[str, "str"], None, None), }, - (Union[str, "str"], True): {"x": str}, - (List["str"], False): {"x": List[str]}, - (List["str"], True): {"x": List[str]}, - (Optional[List[str]], False): {"x": Optional[List[str]]}, - (Optional[List[str]], True): {"x": Optional[List[str]]}, + (Union[str, "str"], True): {'x': str}, + (List["str"], False): {'x': List[str]}, + (List["str"], True): {'x': List[str]}, + (Optional[List[str]], False): {'x': Optional[List[str]]}, + (Optional[List[str]], True): {'x': Optional[List[str]]}, } for (annot, none_default), expected in cases.items(): with self.subTest(annotation=annot, none_default=none_default, expected_type_hints=expected): if annot == (): if none_default: - def func(x = None): pass + def func(x=None): pass else: def func(x): pass elif none_default: From 58082b970bf901b94020ce0d7d68bf2f60c18e31 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Oct 2024 17:56:36 +0200 Subject: [PATCH 06/15] Increase test coverage --- src/test_typing_extensions.py | 77 +++++++++++++++++++++-------------- src/typing_extensions.py | 21 +++++++++- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index f21fdec6..4967a880 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1650,40 +1650,50 @@ def test_annotation_and_optional_default(self): optional_annotation = Optional[annotation] cases = { - # (annotation, none_default) : expected_type_hints - ((), False): {}, + # (annotation, skip_as_str): expected_type_hints + # Should skip_as_str if contains a ForwardRef. ((), True): {}, - (int, False): {'x': int}, - (int, True): {'x': int}, - (Optional[int], False): {'x': Optional[int]}, - (Optional[int], True): {'x': Optional[int]}, - (optional_annotation, False): {'x': optional_annotation}, - (optional_annotation, True): {'x': optional_annotation}, - (str(optional_annotation), True): {'x': optional_annotation}, - (annotation, False): {'x': annotation}, - (annotation, True): {'x': annotation}, - (Union[annotation, T], False): {'x': Union[annotation, T]}, - (Union[annotation, T], True): {'x': Union[annotation, T]}, + (int, True): {"x": int}, + ("int", True): {"x": int}, + (Optional[int], False): {"x": Optional[int]}, + (optional_annotation, False): {"x": optional_annotation}, + (annotation, False): {"x": annotation}, + (Union[annotation, T], True): {"x": Union[annotation, T]}, ("Union[Annotated[Union[int, None], 'data'], T]", True): { - 'x': Union[annotation, T] + "x": Union[annotation, T] }, - (Union[str, None, "str"], False): {'x': Optional[str]}, - (Union[str, None, "str"], True): {'x': Optional[str]}, - (Union[str, "str"], False): { - 'x': str - if sys.version_info >= (3, 9) - # _eval_type does not resolve correctly to str in 3.8 - else typing._eval_type(Union[str, "str"], None, None), + (Union[str, None, str], False): {"x": Optional[str]}, + (Union[str, None, "str"], True): {"x": Optional[str]}, + (Union[str, "str"], True): {"x": str}, + (List["str"], True): {"x": List[str]}, + (Optional[List[str]], False): {"x": Optional[List[str]]}, + (Tuple[Unpack[Tuple[int, str]]], False): { + "x": Tuple[Unpack[Tuple[int, str]]] }, - (Union[str, "str"], True): {'x': str}, - (List["str"], False): {'x': List[str]}, - (List["str"], True): {'x': List[str]}, - (Optional[List[str]], False): {'x': Optional[List[str]]}, - (Optional[List[str]], True): {'x': Optional[List[str]]}, } - - for (annot, none_default), expected in cases.items(): - with self.subTest(annotation=annot, none_default=none_default, expected_type_hints=expected): + for ((annot, skip_as_str), expected), none_default, as_str, wrap_optional in itertools.product( + cases.items(), (False, True), (False, True), (False, True) + ): + if wrap_optional: + if annot == (): + continue + if (get_origin(annot) is not Optional + or (sys.version_info[:2] == (3, 8) and annot._name != "Optional") + ): + annot = Optional[annot] + expected = {"x": Optional[expected['x']]} + if as_str: + if skip_as_str or annot == (): + continue + annot = str(annot) + with self.subTest( + annotation=annot, + as_str=as_str, + none_default=none_default, + expected_type_hints=expected, + wrap_optional=wrap_optional, + ): + # Create function to check if annot == (): if none_default: def func(x=None): pass @@ -1693,7 +1703,14 @@ def func(x): pass def func(x: annot = None): pass else: def func(x: annot): pass - self.assertEqual(get_type_hints(func, include_extras=True), expected) + type_hints = get_type_hints(func, include_extras=True) + self.assertEqual(type_hints, expected) + self.assertEqual(hash(type_hints.values()), hash(expected.values())) + with self.subTest("Test str and repr"): + if sys.version_info[:2] == (3, 8) and annot == Union[str, None, "str"]: + # This also skips Union[str, "str"] wrap_optional=True which has the same problem + self.skipTest("In 3.8 repr is Union[str, None, str]") + self.assertEqual(str(type_hints)+repr(type_hints), str(expected)+repr(type_hints)) class GetUtilitiesTestCase(TestCase): diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 925f9e52..786349a2 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1238,6 +1238,22 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): hint = typing.get_type_hints(obj, globalns=globalns, localns=localns) if sys.version_info < (3, 11): _clean_optional(obj, hint, globalns, localns) + # types from get_type_hints might not be from a cached version + # In 3.8 eval_type does not handle all Optional[ForwardRef] correctly + # this also returns cached versions of Union and Optional + if sys.version_info < (3, 9): + hint = { + k: ( + t + if get_origin(t) not in (Union, Optional) + else ( + Optional[t.__args__[0]] + if get_origin(t) == Optional + else Union[t.__args__] + ) + ) + for k, t in hint.items() + } if include_extras: return hint return {k: _strip_extras(t) for k, t in hint.items()} @@ -1261,7 +1277,7 @@ def _clean_optional(obj, hints, globalns=None, localns=None): # see https://github.com/python/typing_extensions/issues/310 if not hints or isinstance(obj, type): return - defaults = typing._get_defaults(obj) + defaults = typing._get_defaults(obj) # avoid accessing __annotations___ if not defaults: return original_hints = obj.__annotations__ @@ -1300,7 +1316,8 @@ def _clean_optional(obj, hints, globalns=None, localns=None): original_evaluated = typing._eval_type(original_value, globalns, localns) if sys.version_info < (3, 9) and get_origin(original_evaluated) is Union: # Union[str, None, "str"] is not reduced to Union[str, None] - original_evaluated = Union[original_evaluated.__args__] + container = Optional if original_evaluated._name == "Optional" else Union + original_evaluated = container[original_evaluated.__args__] # Compare if values differ if original_evaluated != value: hints[name] = original_evaluated From 5c94bd6f41f1aab178f328cfe7c079ad89acfd25 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Oct 2024 18:02:24 +0200 Subject: [PATCH 07/15] use key access and not values --- src/test_typing_extensions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 4967a880..c9e163aa 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1705,7 +1705,8 @@ def func(x: annot = None): pass def func(x: annot): pass type_hints = get_type_hints(func, include_extras=True) self.assertEqual(type_hints, expected) - self.assertEqual(hash(type_hints.values()), hash(expected.values())) + for k in type_hints.keys(): + self.assertEqual(hash(type_hints[k]), hash(expected[k])) with self.subTest("Test str and repr"): if sys.version_info[:2] == (3, 8) and annot == Union[str, None, "str"]: # This also skips Union[str, "str"] wrap_optional=True which has the same problem From b42e2031df9ec611739568bc80aa95777f0a6373 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Oct 2024 18:06:49 +0200 Subject: [PATCH 08/15] str test passes as well --- src/test_typing_extensions.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index c9e163aa..3552daed 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1707,11 +1707,7 @@ def func(x: annot): pass self.assertEqual(type_hints, expected) for k in type_hints.keys(): self.assertEqual(hash(type_hints[k]), hash(expected[k])) - with self.subTest("Test str and repr"): - if sys.version_info[:2] == (3, 8) and annot == Union[str, None, "str"]: - # This also skips Union[str, "str"] wrap_optional=True which has the same problem - self.skipTest("In 3.8 repr is Union[str, None, str]") - self.assertEqual(str(type_hints)+repr(type_hints), str(expected)+repr(type_hints)) + self.assertEqual(str(type_hints)+repr(type_hints), str(expected)+repr(type_hints)) class GetUtilitiesTestCase(TestCase): From ed552e462ce0cc4530881f1b419a437dd1a8ede9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Oct 2024 19:11:01 +0200 Subject: [PATCH 09/15] removed invalid 3.8 code --- src/typing_extensions.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 786349a2..483066a3 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1238,20 +1238,13 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): hint = typing.get_type_hints(obj, globalns=globalns, localns=localns) if sys.version_info < (3, 11): _clean_optional(obj, hint, globalns, localns) - # types from get_type_hints might not be from a cached version - # In 3.8 eval_type does not handle all Optional[ForwardRef] correctly - # this also returns cached versions of Union and Optional if sys.version_info < (3, 9): + # In 3.8 eval_type does not handle all Optional[ForwardRef] correctly + # this also returns cached versions of Union hint = { - k: ( - t - if get_origin(t) not in (Union, Optional) - else ( - Optional[t.__args__[0]] - if get_origin(t) == Optional - else Union[t.__args__] - ) - ) + k: (t + if get_origin(t) != Union + else Union[t.__args__]) for k, t in hint.items() } if include_extras: @@ -1316,8 +1309,7 @@ def _clean_optional(obj, hints, globalns=None, localns=None): original_evaluated = typing._eval_type(original_value, globalns, localns) if sys.version_info < (3, 9) and get_origin(original_evaluated) is Union: # Union[str, None, "str"] is not reduced to Union[str, None] - container = Optional if original_evaluated._name == "Optional" else Union - original_evaluated = container[original_evaluated.__args__] + original_evaluated = Union[original_evaluated.__args__] # Compare if values differ if original_evaluated != value: hints[name] = original_evaluated From 313ddd83de36002d78c61b18ff53bcc27c398f25 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Oct 2024 23:34:21 +0200 Subject: [PATCH 10/15] Extended tests fixed wrong variable and format --- src/test_typing_extensions.py | 34 ++++++++++++++++++++++++---------- src/typing_extensions.py | 4 ++-- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 3552daed..eb5eef59 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1648,6 +1648,7 @@ def test_final_forward_ref(self): def test_annotation_and_optional_default(self): annotation = Annotated[Union[int, None], "data"] optional_annotation = Optional[annotation] + NoneAlias = None cases = { # (annotation, skip_as_str): expected_type_hints @@ -1667,21 +1668,31 @@ def test_annotation_and_optional_default(self): (Union[str, "str"], True): {"x": str}, (List["str"], True): {"x": List[str]}, (Optional[List[str]], False): {"x": Optional[List[str]]}, - (Tuple[Unpack[Tuple[int, str]]], False): { - "x": Tuple[Unpack[Tuple[int, str]]] - }, + (Unpack[Tuple[int, None]], False): {"x": Unpack[Tuple[int, None]]}, + (Union[str, "Union[int, None]"], True): {"x": Union[str, int, None]}, + (NoneAlias, True): {"x": type(None)}, + ("NoneAlias", True): {"x": type(None)}, + (Union[str, "NoneAlias"], True): {"x": Optional[str]}, } + if sys.version_info >= (3, 10): # cannot construct UnionTypes + cases_3_10 = { + ("str | NoneAlias", True) : {"x": str | None}, + (str | None, False) : {"x": Optional[str]}, + } + cases.update(cases_3_10) for ((annot, skip_as_str), expected), none_default, as_str, wrap_optional in itertools.product( cases.items(), (False, True), (False, True), (False, True) ): + # Special case: + skip_reason = None + if sys.version_info[:2] == (3, 10) and annot == "str | NoneAlias" and none_default: + # Optional[str | None] -> Optional[str] not UnionType + skip_reason = "UnionType not preserved in 3.10" if wrap_optional: if annot == (): continue - if (get_origin(annot) is not Optional - or (sys.version_info[:2] == (3, 8) and annot._name != "Optional") - ): - annot = Optional[annot] - expected = {"x": Optional[expected['x']]} + annot = Optional[annot] + expected = {"x": Optional[expected['x']]} if as_str: if skip_as_str or annot == (): continue @@ -1703,11 +1714,14 @@ def func(x): pass def func(x: annot = None): pass else: def func(x: annot): pass - type_hints = get_type_hints(func, include_extras=True) + type_hints = get_type_hints(func, globals(), locals(), include_extras=True) self.assertEqual(type_hints, expected) for k in type_hints.keys(): self.assertEqual(hash(type_hints[k]), hash(expected[k])) - self.assertEqual(str(type_hints)+repr(type_hints), str(expected)+repr(type_hints)) + with self.subTest("Check str and repr"): + if skip_reason == "UnionType not preserved in 3.10": + self.skipTest(skip_reason) + self.assertEqual(str(type_hints) + repr(type_hints), str(expected) + repr(expected)) class GetUtilitiesTestCase(TestCase): diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 483066a3..74bfaa22 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1239,8 +1239,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): if sys.version_info < (3, 11): _clean_optional(obj, hint, globalns, localns) if sys.version_info < (3, 9): - # In 3.8 eval_type does not handle all Optional[ForwardRef] correctly - # this also returns cached versions of Union + # In 3.8 eval_type does not flatten Optional[ForwardRef] correctly + # This will recreate and and cache Unions. hint = { k: (t if get_origin(t) != Union From 3ba4ee72db6b256243f568601e8e0c7f62fe0ec3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 11 Oct 2024 16:57:56 +0200 Subject: [PATCH 11/15] refinement of tests, more tests with aliases --- src/test_typing_extensions.py | 83 +++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index eb5eef59..2d00fdfc 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1647,65 +1647,71 @@ def test_final_forward_ref(self): def test_annotation_and_optional_default(self): annotation = Annotated[Union[int, None], "data"] - optional_annotation = Optional[annotation] NoneAlias = None + StrAlias = str + T_default = TypeVar("T_default", default=None) + Ts = TypeVarTuple("Ts") cases = { - # (annotation, skip_as_str): expected_type_hints - # Should skip_as_str if contains a ForwardRef. - ((), True): {}, - (int, True): {"x": int}, - ("int", True): {"x": int}, - (Optional[int], False): {"x": Optional[int]}, - (optional_annotation, False): {"x": optional_annotation}, - (annotation, False): {"x": annotation}, - (Union[annotation, T], True): {"x": Union[annotation, T]}, - ("Union[Annotated[Union[int, None], 'data'], T]", True): { - "x": Union[annotation, T] - }, - (Union[str, None, str], False): {"x": Optional[str]}, - (Union[str, None, "str"], True): {"x": Optional[str]}, - (Union[str, "str"], True): {"x": str}, - (List["str"], True): {"x": List[str]}, - (Optional[List[str]], False): {"x": Optional[List[str]]}, - (Unpack[Tuple[int, None]], False): {"x": Unpack[Tuple[int, None]]}, - (Union[str, "Union[int, None]"], True): {"x": Union[str, int, None]}, - (NoneAlias, True): {"x": type(None)}, - ("NoneAlias", True): {"x": type(None)}, - (Union[str, "NoneAlias"], True): {"x": Optional[str]}, + # annotation: expected_type_hints + Annotated[None, "none"] : Annotated[None, "none"], + annotation : annotation, + Optional[int] : Optional[int], + Optional[List[str]] : Optional[List[str]], + Optional[annotation] : Optional[annotation], + Union[str, None, str] : Optional[str], + Unpack[Tuple[int, None]]: Unpack[Tuple[int, None]], + # Note: A starred *Ts will use typing.Unpack in 3.11+ see Issue #485 + Unpack[Ts] : Unpack[Ts], } - if sys.version_info >= (3, 10): # cannot construct UnionTypes - cases_3_10 = { - ("str | NoneAlias", True) : {"x": str | None}, - (str | None, False) : {"x": Optional[str]}, - } - cases.update(cases_3_10) - for ((annot, skip_as_str), expected), none_default, as_str, wrap_optional in itertools.product( + # contains a ForwardRef, TypeVar(~prefix) or no expression + do_not_stringify_cases = { + () : {}, + int : int, + "int" : int, + None : type(None), + "NoneAlias" : type(None), + List["str"] : List[str], + Union[str, "str"] : str, + Union[str, None, "str"] : Optional[str], + Union[str, "NoneAlias", "StrAlias"]: Optional[str], + Union[str, "Union[None, StrAlias]"]: Optional[str], + Union["annotation", T_default] : Union[annotation, T_default], + Annotated["annotation", "nested"] : Annotated[Union[int, None], "data", "nested"], + } + if TYPING_3_10_0: # cannot construct UnionTypes + do_not_stringify_cases["str | NoneAlias | StrAlias"] = str | None + cases[str | None] = Optional[str] + cases.update(do_not_stringify_cases) + for (annot, expected), none_default, as_str, wrap_optional in itertools.product( cases.items(), (False, True), (False, True), (False, True) ): # Special case: skip_reason = None - if sys.version_info[:2] == (3, 10) and annot == "str | NoneAlias" and none_default: - # Optional[str | None] -> Optional[str] not UnionType + annot_unchanged = annot + if sys.version_info[:2] == (3, 10) and annot == "str | NoneAlias | StrAlias" and none_default: + # different repr here as Optional[str | None] -> Optional[str] not a UnionType skip_reason = "UnionType not preserved in 3.10" if wrap_optional: - if annot == (): + if annot_unchanged == (): continue annot = Optional[annot] - expected = {"x": Optional[expected['x']]} + expected = {"x": Optional[expected]} + else: + expected = {"x": expected} if annot_unchanged != () else {} if as_str: - if skip_as_str or annot == (): + if annot_unchanged in do_not_stringify_cases or annot_unchanged == (): continue annot = str(annot) with self.subTest( annotation=annot, as_str=as_str, + wrap_optional=wrap_optional, none_default=none_default, expected_type_hints=expected, - wrap_optional=wrap_optional, ): # Create function to check - if annot == (): + if annot_unchanged == (): if none_default: def func(x=None): pass else: @@ -1715,9 +1721,12 @@ def func(x: annot = None): pass else: def func(x: annot): pass type_hints = get_type_hints(func, globals(), locals(), include_extras=True) + # Equality self.assertEqual(type_hints, expected) + # Hash for k in type_hints.keys(): self.assertEqual(hash(type_hints[k]), hash(expected[k])) + # Repr with self.subTest("Check str and repr"): if skip_reason == "UnionType not preserved in 3.10": self.skipTest(skip_reason) From 52c93e9d9269414dc03210ed52e1ac961110241d Mon Sep 17 00:00:00 2001 From: Daraan Date: Mon, 21 Oct 2024 15:10:15 +0200 Subject: [PATCH 12/15] Add comment for special case Co-authored-by: Jelle Zijlstra --- src/test_typing_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 2d00fdfc..d99fa5dd 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1666,7 +1666,7 @@ def test_annotation_and_optional_default(self): } # contains a ForwardRef, TypeVar(~prefix) or no expression do_not_stringify_cases = { - () : {}, + () : {}, # Special-cased below to create an unannotated parameter int : int, "int" : int, None : type(None), From a1777f317c6e06d1be3ee0791a4b6000f4336512 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 21 Oct 2024 16:29:29 +0200 Subject: [PATCH 13/15] Assure UnionType stays UnionType; check repr only + updated comments --- src/test_typing_extensions.py | 8 +++++--- src/typing_extensions.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index d99fa5dd..97f782d1 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1679,7 +1679,7 @@ def test_annotation_and_optional_default(self): Union["annotation", T_default] : Union[annotation, T_default], Annotated["annotation", "nested"] : Annotated[Union[int, None], "data", "nested"], } - if TYPING_3_10_0: # cannot construct UnionTypes + if TYPING_3_10_0: # cannot construct UnionTypes before 3.10 do_not_stringify_cases["str | NoneAlias | StrAlias"] = str | None cases[str | None] = Optional[str] cases.update(do_not_stringify_cases) @@ -1690,7 +1690,7 @@ def test_annotation_and_optional_default(self): skip_reason = None annot_unchanged = annot if sys.version_info[:2] == (3, 10) and annot == "str | NoneAlias | StrAlias" and none_default: - # different repr here as Optional[str | None] -> Optional[str] not a UnionType + # In 3.10 converts Optional[str | None] to Optional[str] which has a different repr skip_reason = "UnionType not preserved in 3.10" if wrap_optional: if annot_unchanged == (): @@ -1726,11 +1726,13 @@ def func(x: annot): pass # Hash for k in type_hints.keys(): self.assertEqual(hash(type_hints[k]), hash(expected[k])) + # Test if UnionTypes are preserved + self.assertEqual(isinstance(type_hints[k], type(expected[k])), True) # Repr with self.subTest("Check str and repr"): if skip_reason == "UnionType not preserved in 3.10": self.skipTest(skip_reason) - self.assertEqual(str(type_hints) + repr(type_hints), str(expected) + repr(expected)) + self.assertEqual(repr(type_hints), repr(expected)) class GetUtilitiesTestCase(TestCase): diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 74bfaa22..2d1c4aa4 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1282,7 +1282,7 @@ def _clean_optional(obj, hints, globalns=None, localns=None): ): continue original_value = original_hints[name] - if original_value is None: # should not happen + if original_value is None: # should be NoneType already; check just in case original_value = _NoneType # Forward reference if isinstance(original_value, str): @@ -1310,8 +1310,14 @@ def _clean_optional(obj, hints, globalns=None, localns=None): if sys.version_info < (3, 9) and get_origin(original_evaluated) is Union: # Union[str, None, "str"] is not reduced to Union[str, None] original_evaluated = Union[original_evaluated.__args__] - # Compare if values differ - if original_evaluated != value: + # Compare if values differ. Note that even if equal + # value might be cached by typing._tp_cache contrary to original_evaluated + if original_evaluated != value or ( + # 3.10: ForwardRefs of UnionType might be turned into _UnionGenericAlias + hasattr(_types, "UnionType") + and isinstance(original_evaluated, _types.UnionType) + and not isinstance(value, _types.UnionType) + ): hints[name] = original_evaluated # Python 3.9+ has PEP 593 (Annotated) From 9e59796f453cdeec7d85e2b247c3a7f34ff45c7a Mon Sep 17 00:00:00 2001 From: Daraan Date: Mon, 21 Oct 2024 18:57:16 +0200 Subject: [PATCH 14/15] Update src/test_typing_extensions.py Co-authored-by: Jelle Zijlstra --- src/test_typing_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 97f782d1..31c7aef5 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -1727,7 +1727,7 @@ def func(x: annot): pass for k in type_hints.keys(): self.assertEqual(hash(type_hints[k]), hash(expected[k])) # Test if UnionTypes are preserved - self.assertEqual(isinstance(type_hints[k], type(expected[k])), True) + self.assertIs(type(type_hints[k]), type(expected[k])) # Repr with self.subTest("Check str and repr"): if skip_reason == "UnionType not preserved in 3.10": From 54b8eb0bf9d0f0dfcacb4583be87ef5ad49b2096 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 21 Oct 2024 20:06:32 +0200 Subject: [PATCH 15/15] Corrected and clarified case --- src/typing_extensions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 838c273a..d7363ddb 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -1282,7 +1282,8 @@ def _clean_optional(obj, hints, globalns=None, localns=None): ): continue original_value = original_hints[name] - if original_value is None: # should be NoneType already; check just in case + # value=NoneType should have caused a skip above but check for safety + if original_value is None: original_value = _NoneType # Forward reference if isinstance(original_value, str):