From 17d3e15b10564c89e9c02e56957271a77a25e5b6 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 10 Jan 2025 18:33:27 -0800 Subject: [PATCH] add tests for step-1 of overload evaluation --- .../results/mypy/overloads_evaluation.toml | 25 +++++++++++ .../results/pyre/overloads_evaluation.toml | 9 ++++ conformance/results/pyre/version.toml | 2 +- .../results/pyright/overloads_evaluation.toml | 12 +++++ .../results/pytype/overloads_evaluation.toml | 44 +++++++++++++++++++ conformance/results/pytype/version.toml | 2 +- conformance/results/results.html | 10 ++++- conformance/tests/overloads_evaluation.py | 44 +++++++++++++++++++ 8 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 conformance/results/mypy/overloads_evaluation.toml create mode 100644 conformance/results/pyre/overloads_evaluation.toml create mode 100644 conformance/results/pyright/overloads_evaluation.toml create mode 100644 conformance/results/pytype/overloads_evaluation.toml create mode 100644 conformance/tests/overloads_evaluation.py diff --git a/conformance/results/mypy/overloads_evaluation.toml b/conformance/results/mypy/overloads_evaluation.toml new file mode 100644 index 00000000..ed5346a5 --- /dev/null +++ b/conformance/results/mypy/overloads_evaluation.toml @@ -0,0 +1,25 @@ +conformant = "Partial" +notes = """ +Does not pick a winning overload based on arity, prior to considering argument types. +""" +conformance_automated = "Fail" +errors_diff = """ +Line 36: Unexpected errors ['overloads_evaluation.py:36: error: Expression is of type "Any", not "int" [assert-type]'] +Line 42: Unexpected errors ['overloads_evaluation.py:42: error: Expression is of type "Any", not "str" [assert-type]'] +""" +output = """ +overloads_evaluation.py:26: error: All overload variants of "num_args" require at least one argument [call-overload] +overloads_evaluation.py:26: note: Possible overload variants: +overloads_evaluation.py:26: note: def num_args(x: int, y: str) -> int +overloads_evaluation.py:26: note: def num_args(x: str) -> str +overloads_evaluation.py:35: error: No overload variant of "num_args" matches argument types "int", "int" [call-overload] +overloads_evaluation.py:35: note: Possible overload variants: +overloads_evaluation.py:35: note: def num_args(x: int, y: str) -> int +overloads_evaluation.py:35: note: def num_args(x: str) -> str +overloads_evaluation.py:36: error: Expression is of type "Any", not "int" [assert-type] +overloads_evaluation.py:41: error: No overload variant of "num_args" matches argument type "int" [call-overload] +overloads_evaluation.py:41: note: Possible overload variants: +overloads_evaluation.py:41: note: def num_args(x: int, y: str) -> int +overloads_evaluation.py:41: note: def num_args(x: str) -> str +overloads_evaluation.py:42: error: Expression is of type "Any", not "str" [assert-type] +""" diff --git a/conformance/results/pyre/overloads_evaluation.toml b/conformance/results/pyre/overloads_evaluation.toml new file mode 100644 index 00000000..311020b4 --- /dev/null +++ b/conformance/results/pyre/overloads_evaluation.toml @@ -0,0 +1,9 @@ +conformant = "Pass" +conformance_automated = "Pass" +errors_diff = """ +""" +output = """ +overloads_evaluation.py:26:0 Missing argument [20]: Call `num_args` expects argument `x`. +overloads_evaluation.py:35:19 Incompatible parameter type [6]: In call `num_args`, for 2nd positional argument, expected `str` but got `int`. +overloads_evaluation.py:41:16 Incompatible parameter type [6]: In call `num_args`, for 1st positional argument, expected `str` but got `int`. +""" diff --git a/conformance/results/pyre/version.toml b/conformance/results/pyre/version.toml index cda658b7..cd4e9e46 100644 --- a/conformance/results/pyre/version.toml +++ b/conformance/results/pyre/version.toml @@ -1,2 +1,2 @@ version = "pyre 0.9.23" -test_duration = 6.6 +test_duration = 6.7 diff --git a/conformance/results/pyright/overloads_evaluation.toml b/conformance/results/pyright/overloads_evaluation.toml new file mode 100644 index 00000000..9ef2de0b --- /dev/null +++ b/conformance/results/pyright/overloads_evaluation.toml @@ -0,0 +1,12 @@ +conformant = "Pass" +conformance_automated = "Pass" +errors_diff = """ +""" +output = """ +overloads_evaluation.py:26:1 - error: No overloads for "num_args" match the provided arguments +  Argument types: () (reportCallIssue) +overloads_evaluation.py:35:20 - error: Argument of type "Literal[1]" cannot be assigned to parameter "y" of type "str" in function "num_args" +  "Literal[1]" is not assignable to "str" (reportArgumentType) +overloads_evaluation.py:41:17 - error: Argument of type "Literal[1]" cannot be assigned to parameter "x" of type "str" in function "num_args" +  "Literal[1]" is not assignable to "str" (reportArgumentType) +""" diff --git a/conformance/results/pytype/overloads_evaluation.toml b/conformance/results/pytype/overloads_evaluation.toml new file mode 100644 index 00000000..5ae1b37d --- /dev/null +++ b/conformance/results/pytype/overloads_evaluation.toml @@ -0,0 +1,44 @@ +conformant = "Partial" +notes = """ +Does not pick a winning overload based on arity, prior to considering argument types. +""" +conformance_automated = "Fail" +errors_diff = """ +Line 24: Unexpected errors ['overloads_evaluation.py:24:12: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in num_args: bad return type [bad-return-type]'] +Line 36: Unexpected errors ['overloads_evaluation.py:36:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in : Any [assert-type]'] +Line 42: Unexpected errors ['overloads_evaluation.py:42:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in : Any [assert-type]'] +""" +output = """ +overloads_evaluation.py:24:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in num_args: bad return type [bad-return-type] + + return 1 + \u001b[1m\u001b[31m~\u001b[39m\u001b[0m + +Called from (traceback): + line 38, in current file +overloads_evaluation.py:26:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in : Missing parameter 'x' in call to function num_args [missing-parameter] + +num_args() # E: no matching overload +\u001b[1m\u001b[31m~~~~~~~~~~\u001b[39m\u001b[0m + +overloads_evaluation.py:35:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in : Function num_args was called with the wrong arguments [wrong-arg-types] + +ret2 = num_args(1, 1) # E: Literal[1] not assignable to str + \u001b[1m\u001b[31m~~~~~~~~~~~~~~\u001b[39m\u001b[0m + +overloads_evaluation.py:36:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in : Any [assert-type] + +assert_type(ret2, int) +\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m + +overloads_evaluation.py:41:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in : Function num_args was called with the wrong arguments [wrong-arg-types] + +ret4 = num_args(1) # E: Literal[1] not assignable to str + \u001b[1m\u001b[31m~~~~~~~~~~~\u001b[39m\u001b[0m + +overloads_evaluation.py:42:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in : Any [assert-type] + +assert_type(ret4, str) +\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m + +""" diff --git a/conformance/results/pytype/version.toml b/conformance/results/pytype/version.toml index 7dff3b18..add97f41 100644 --- a/conformance/results/pytype/version.toml +++ b/conformance/results/pytype/version.toml @@ -1,2 +1,2 @@ version = "pytype 2024.10.11" -test_duration = 30.1 +test_duration = 30.7 diff --git a/conformance/results/results.html b/conformance/results/results.html index 71692ca0..414eee34 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -165,10 +165,10 @@

Python Type System Conformance Test Results

1.5sec
pyre 0.9.23
-
6.6sec
+
6.7sec
pytype 2024.10.11
-
30.1sec
+
30.7sec
@@ -690,6 +690,12 @@

Python Type System Conformance Test Results

Partial

Expects @final and @override to be present on all overloads, not just first.

Fail

Does not enforce any of the specified rules regarding overload definitions.

+     overloads_evaluation +
Partial

Does not pick a winning overload based on arity, prior to considering argument types.

+Pass +Pass +
Partial

Does not pick a winning overload based on arity, prior to considering argument types.

+      overloads_overlap Pass Pass diff --git a/conformance/tests/overloads_evaluation.py b/conformance/tests/overloads_evaluation.py new file mode 100644 index 00000000..2f69a4a8 --- /dev/null +++ b/conformance/tests/overloads_evaluation.py @@ -0,0 +1,44 @@ +""" +Tests for evaluation of calls to overloaded functions. +""" + +from typing import assert_type, overload + + +# > Step 1: Examine the argument list to determine the number of +# > positional and keyword arguments. Use this information to eliminate any +# > overload candidates that are not plausible based on their +# > input signatures. + +# > - If no candidate overloads remain, generate an error and stop. + +@overload +def num_args(x: int, y: str) -> int: + ... + +@overload +def num_args(x: str) -> str: + ... + +def num_args(x: int | str, y: str = "") -> int | str: + return 1 + +num_args() # E: no matching overload + + +# > - If only one candidate overload remains, it is the winning match. Evaluate +# > it as if it were a non-overloaded function call and stop. + +ret1 = num_args(1, "") +assert_type(ret1, int) + +ret2 = num_args(1, 1) # E: Literal[1] not assignable to str +assert_type(ret2, int) + +ret3 = num_args("") +assert_type(ret3, str) + +ret4 = num_args(1) # E: Literal[1] not assignable to str +assert_type(ret4, str) + +