From 6674c63dc7bb175acc997ddcb799e8dbbafd2968 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 13 Jun 2024 21:01:05 +0200 Subject: [PATCH 01/88] Add codeowner for Makefile.pre.in and Modules/Setup* (#120468) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8bc40fcb9e8999..1f9047ab97e934 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,8 @@ # Build system configure* @erlend-aasland @corona10 +Makefile.pre.in @erlend-aasland +Modules/Setup* @erlend-aasland # asyncio **/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc From a3711afefa7a520b3de01be3b2367cb830d1fc84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:03:01 +0200 Subject: [PATCH 02/88] gh-120012: clarify the behaviour of `multiprocessing.Queue.empty` on closed queues. (GH-120102) * improve doc for `multiprocessing.Queue.empty` * add tests for checking emptiness of queues Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.rst | 4 +++ Lib/test/_test_multiprocessing.py | 26 +++++++++++++++++++ ...-06-05-12-36-18.gh-issue-120012.f14DbQ.rst | 3 +++ 3 files changed, 33 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 49762491bae5f4..426291c5f0743d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -837,6 +837,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. Because of multithreading/multiprocessing semantics, this is not reliable. + May raise an :exc:`OSError` on closed queues. (not guaranteed) + .. method:: full() Return ``True`` if the queue is full, ``False`` otherwise. Because of @@ -940,6 +942,8 @@ For an example of the usage of queues for interprocess communication see Return ``True`` if the queue is empty, ``False`` otherwise. + Always raises an :exc:`OSError` if the SimpleQueue is closed. + .. method:: get() Remove and return an item from the queue. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 301541a666e140..4b3a0645cfc84a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1332,6 +1332,23 @@ def _on_queue_feeder_error(e, obj): self.assertTrue(not_serializable_obj.reduce_was_called) self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_closed_queue_put_get_exceptions(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.close() @@ -5815,6 +5832,15 @@ def _test_empty(cls, queue, child_can_start, parent_can_continue): finally: parent_can_continue.set() + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() diff --git a/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst b/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst new file mode 100644 index 00000000000000..2bf0c977b90387 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-06-05-12-36-18.gh-issue-120012.f14DbQ.rst @@ -0,0 +1,3 @@ +Clarify the behaviours of :meth:`multiprocessing.Queue.empty` and +:meth:`multiprocessing.SimpleQueue.empty` on closed queues. +Patch by Bénédikt Tran. From d88a1f2e156cd1072119afa91d4f4dc4037c1b21 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 13 Jun 2024 21:25:26 +0100 Subject: [PATCH 03/88] GH-119054: Add "Renaming and deleting" section to pathlib docs. (#120465) Add dedicated subsection for `pathlib.Path.rename()`, `replace()`, `unlink()` and `rmdir()`. --- Doc/library/pathlib.rst | 124 +++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 138e41404dec9c..278851549c6c3b 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1429,6 +1429,70 @@ Creating files and directories available. In previous versions, :exc:`NotImplementedError` was raised. +Renaming and deleting +^^^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.rename(target) + + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. On Unix, if *target* exists + and is a file, it will be replaced silently if the user has permission. + On Windows, if *target* exists, :exc:`FileExistsError` will be raised. + *target* can be either a string or another path object:: + + >>> p = Path('foo') + >>> p.open('w').write('some text') + 9 + >>> target = Path('bar') + >>> p.rename(target) + PosixPath('bar') + >>> target.open().read() + 'some text' + + The target path may be absolute or relative. Relative paths are interpreted + relative to the current working directory, *not* the directory of the + :class:`!Path` object. + + It is implemented in terms of :func:`os.rename` and gives the same guarantees. + + .. versionchanged:: 3.8 + Added return value, return the new :class:`!Path` instance. + + +.. method:: Path.replace(target) + + Rename this file or directory to the given *target*, and return a new + :class:`!Path` instance pointing to *target*. If *target* points to an + existing file or empty directory, it will be unconditionally replaced. + + The target path may be absolute or relative. Relative paths are interpreted + relative to the current working directory, *not* the directory of the + :class:`!Path` object. + + .. versionchanged:: 3.8 + Added return value, return the new :class:`!Path` instance. + + +.. method:: Path.unlink(missing_ok=False) + + Remove this file or symbolic link. If the path points to a directory, + use :func:`Path.rmdir` instead. + + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. + + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). + + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. + + +.. method:: Path.rmdir() + + Remove this directory. The directory must be empty. + + Other methods ^^^^^^^^^^^^^ @@ -1545,47 +1609,6 @@ Other methods available. In previous versions, :exc:`NotImplementedError` was raised. -.. method:: Path.rename(target) - - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. - On Windows, if *target* exists, :exc:`FileExistsError` will be raised. - *target* can be either a string or another path object:: - - >>> p = Path('foo') - >>> p.open('w').write('some text') - 9 - >>> target = Path('bar') - >>> p.rename(target) - PosixPath('bar') - >>> target.open().read() - 'some text' - - The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. - - It is implemented in terms of :func:`os.rename` and gives the same guarantees. - - .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - -.. method:: Path.replace(target) - - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. If *target* points to an existing file or - empty directory, it will be unconditionally replaced. - - The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. - - .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - .. method:: Path.absolute() Make the path absolute, without normalization or resolving symlinks. @@ -1628,25 +1651,6 @@ Other methods strict mode, and no exception is raised in non-strict mode. In previous versions, :exc:`RuntimeError` is raised no matter the value of *strict*. -.. method:: Path.rmdir() - - Remove this directory. The directory must be empty. - - -.. method:: Path.unlink(missing_ok=False) - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. - - If *missing_ok* is false (the default), :exc:`FileNotFoundError` is - raised if the path does not exist. - - If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be - ignored (same behavior as the POSIX ``rm -f`` command). - - .. versionchanged:: 3.8 - The *missing_ok* parameter was added. - .. _pathlib-pattern-language: From 42351c3b9a357ec67135b30ed41f59e6f306ac52 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 13 Jun 2024 22:16:40 +0100 Subject: [PATCH 04/88] gh-114053: Fix bad interaction of PEP 695, PEP 563 and `inspect.get_annotations` (#120270) --- Lib/inspect.py | 8 +- .../inspect_stringized_annotations_pep695.py | 72 ++++++++++++ Lib/test/test_inspect/test_inspect.py | 103 ++++++++++++++++++ ...-06-08-15-15-29.gh-issue-114053.WQLAFG.rst | 4 + 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_inspect/inspect_stringized_annotations_pep695.py create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 5570a43ebfea19..11544b8d0d4932 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -274,7 +274,13 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): if globals is None: globals = obj_globals if locals is None: - locals = obj_locals + locals = obj_locals or {} + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + locals = {param.__name__: param for param in type_params} | locals return_value = {key: value if not isinstance(value, str) else eval(value, globals, locals) diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py new file mode 100644 index 00000000000000..723822f8eaa92d --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable, Unpack + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: Unpack[Ts], z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + E_annotations=get_annotations(E, eval_str=True), + E_meth_annotations=get_annotations(E.generic_method, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 0a4fa9343f15e0..140efac530afb2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -22,6 +22,7 @@ import types import tempfile import textwrap +from typing import Unpack import unicodedata import unittest import unittest.mock @@ -47,6 +48,7 @@ from test.test_inspect import inspect_stock_annotations from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 # Functions tested in this suite: @@ -1692,6 +1694,107 @@ def wrapper(a, b): self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = inspect.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = inspect.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = inspect.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = inspect.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = inspect.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ) + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.E_annotations.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.E_meth_annotations.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.E_meth_annotations.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.E_meth_annotations.values()).intersection(results.E.__type_params__), + set() + ) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__) + ) + class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): diff --git a/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst b/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst new file mode 100644 index 00000000000000..be49577a712867 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-15-15-29.gh-issue-114053.WQLAFG.rst @@ -0,0 +1,4 @@ +Fix erroneous :exc:`NameError` when calling :func:`inspect.get_annotations` +with ``eval_str=True``` on a class that made use of :pep:`695` type +parameters in a module that had ``from __future__ import annotations`` at +the top of the file. Patch by Alex Waygood. From 41554ef0e0925695544d96a7bc49af1428d6bb6b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Jun 2024 10:21:35 -0500 Subject: [PATCH 05/88] Stronger tests for the statistics kernel formulas (gh-120506) --- Lib/test/test_statistics.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0b28459f03d86a..c374c947e02a6b 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2434,18 +2434,22 @@ def integrate(func, low, high, steps=10_000): data.append(100) self.assertGreater(f_hat(100), 0.0) - def test_kde_kernel_invcdfs(self): + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() kernel_specs = statistics._kernel_specs - kde = statistics.kde # Verify that cdf / invcdf will round trip xarr = [i/100 for i in range(-100, 101)] + parr = [i/1000 + 5/10000 for i in range(1000)] for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] invcdf = spec['invcdf'] with self.subTest(kernel=kernel): - cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) for x in xarr: self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) @support.requires_resource('cpu') def test_kde_random(self): From 27419f1fce05a18384e6fb3b8ad59b7f532821e6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Jun 2024 11:00:46 -0500 Subject: [PATCH 06/88] Update tests for the itertools docs rough equivalents (#120509) --- Lib/test/test_itertools.py | 333 +++++++++++++++++++++++++++++++++++-- 1 file changed, 315 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 53b8064c3cfe82..5fd6ecf37427f7 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1587,27 +1587,169 @@ def batched_recipe(iterable, n): self.assertEqual(r1, r2) self.assertEqual(e1, e2) + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -1627,6 +1769,161 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + iterator = iter(iterable) + shared_link = [None, None] + return tuple(_tee(iterator, shared_link) for _ in range(n)) + + def _tee(iterator, link): + try: + while True: + if link[1] is None: + link[0] = next(iterator) + link[1] = [None, None] + value, link = link + yield value + except StopIteration: + return + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # Tests not applicable to the tee() recipe + if False: + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # Tests not applicable to the tee() recipe + if False: + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + + # Tests not applicable to the tee() recipe + if False: + # tee pass-through to copyable iterator + a, b = tee('abc') + c, d = tee(a) + self.assertTrue(a is c) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): From 2bacc2343c24c49292dea3461f6b7664fc2d33e2 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 15 Jun 2024 00:10:18 +0800 Subject: [PATCH 07/88] gh-117657: Add TSAN suppression for set_default_allocator_unlocked (#120500) Add TSAN suppression for set_default_allocator_unlocked --- Tools/tsan/suppressions_free_threading.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 05ceaf438b6353..4c8b0b8abd2963 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -63,6 +63,8 @@ race_top:tstate_is_freed race_top:type_modified_unlocked race_top:write_thread_id race_top:PyThreadState_Clear +# Only seen on macOS, sample: https://gist.github.com/aisk/dda53f5d494a4556c35dde1fce03259c +race_top:set_default_allocator_unlocked # https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 thread:pthread_create From 7c38097add9cc24e9f68414cd3e5e1b6cbe38a17 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 14 Jun 2024 17:15:49 +0100 Subject: [PATCH 08/88] GH-73991: Add `pathlib.Path.copy()` (#119058) Add a `Path.copy()` method that copies the content of one file to another. This method is similar to `shutil.copyfile()` but differs in the following ways: - Uses `fcntl.FICLONE` where available (see GH-81338) - Uses `os.copy_file_range` where available (see GH-81340) - Uses `_winapi.CopyFile2` where available, even though this copies more metadata than the other implementations. This makes `WindowsPath.copy()` more similar to `shutil.copy2()`. The method is presently _less_ specified than the `shutil` functions to allow OS-specific optimizations that might copy more or less metadata. Incorporates code from GH-81338 and GH-93152. Co-authored-by: Eryk Sun --- Doc/library/pathlib.rst | 18 ++- Doc/whatsnew/3.14.rst | 7 + Lib/pathlib/_abc.py | 30 ++++ Lib/pathlib/_local.py | 16 ++ Lib/pathlib/_os.py | 138 ++++++++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 62 ++++++++ ...4-05-15-01-36-08.gh-issue-73991.CGknDf.rst | 2 + 7 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 Lib/pathlib/_os.py create mode 100644 Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 278851549c6c3b..c8a3272d7bab4c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1429,8 +1429,22 @@ Creating files and directories available. In previous versions, :exc:`NotImplementedError` was raised. -Renaming and deleting -^^^^^^^^^^^^^^^^^^^^^ +Copying, renaming and deleting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: Path.copy(target) + + Copy the contents of this file to the *target* file. If *target* specifies + a file that already exists, it will be replaced. + + .. note:: + This method uses operating system functionality to copy file content + efficiently. The OS might also copy some metadata, such as file + permissions. After the copy is complete, users may wish to call + :meth:`Path.chmod` to set the permissions of the target file. + + .. versionadded:: 3.14 + .. method:: Path.rename(target) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b357553735e8bb..a102af13a08362 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -100,6 +100,13 @@ os by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) +pathlib +------- + +* Add :meth:`pathlib.Path.copy`, which copies the content of one file to + another, like :func:`shutil.copyfile`. + (Contributed by Barney Gale in :gh:`73991`.) + symtable -------- diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index ecea8e88d1a2e3..586145ead384ea 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -16,6 +16,7 @@ import posixpath from glob import _GlobberBase, _no_recurse_symlinks from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from ._os import copyfileobj __all__ = ["UnsupportedOperation"] @@ -563,6 +564,15 @@ def samefile(self, other_path): return (st.st_ino == other_st.st_ino and st.st_dev == other_st.st_dev) + def _samefile_safe(self, other_path): + """ + Like samefile(), but returns False rather than raising OSError. + """ + try: + return self.samefile(other_path) + except (OSError, ValueError): + return False + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -780,6 +790,26 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + def copy(self, target): + """ + Copy the contents of this file to the given target. + """ + if not isinstance(target, PathBase): + target = self.with_segments(target) + if self._samefile_safe(target): + raise OSError(f"{self!r} and {target!r} are the same file") + with self.open('rb') as source_f: + try: + with target.open('wb') as target_f: + copyfileobj(source_f, target_f) + except IsADirectoryError as e: + if not target.exists(): + # Raise a less confusing exception. + raise FileNotFoundError( + f'Directory does not exist: {target}') from e + else: + raise + def rename(self, target): """ Rename this path to the target path. diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 473fd525768b50..cffed10dbd1207 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -18,6 +18,7 @@ grp = None from ._abc import UnsupportedOperation, PurePathBase, PathBase +from ._os import copyfile __all__ = [ @@ -780,6 +781,21 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): if not exist_ok or not self.is_dir(): raise + if copyfile: + def copy(self, target): + """ + Copy the contents of this file to the given target. + """ + try: + target = os.fspath(target) + except TypeError: + if isinstance(target, PathBase): + # Target is an instance of PathBase but not os.PathLike. + # Use generic implementation from PathBase. + return PathBase.copy(self, target) + raise + copyfile(os.fspath(self), target) + def chmod(self, mode, *, follow_symlinks=True): """ Change the permissions of the path, like os.chmod(). diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000000..1771d54e4167c1 --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,138 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import EBADF, EOPNOTSUPP, ETXTBSY, EXDEV +import os +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def clonefd(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + clonefd = None + + +if posix and hasattr(posix, '_fcopyfile'): + def copyfd(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +elif hasattr(os, 'copy_file_range'): + def copyfd(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +elif hasattr(os, 'sendfile'): + def copyfd(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + copyfd = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if clonefd: + try: + clonefd(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if copyfd: + copyfd(source_fd, target_fd) + return + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 57cc1612c03468..fd71284159d5c0 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1696,6 +1696,68 @@ def test_write_text_with_newlines(self): self.assertEqual((p / 'fileA').read_bytes(), b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') + def test_copy_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_directory(self): + base = self.cls(self.base) + source = base / 'dirA' + target = base / 'copyA' + with self.assertRaises(OSError): + source.copy(target) + + @needs_symlinks + def test_copy_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_to_existing_file(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' / 'fileB' + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + def test_copy_to_existing_directory(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirA' + with self.assertRaises(OSError): + source.copy(target) + + @needs_symlinks + def test_copy_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + source.copy(target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + def test_copy_empty(self): + base = self.cls(self.base) + source = base / 'empty' + target = base / 'copyA' + source.write_bytes(b'') + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(target.read_bytes(), b'') + def test_iterdir(self): P = self.cls p = P(self.base) diff --git a/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst b/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst new file mode 100644 index 00000000000000..c2953c65b2720f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-15-01-36-08.gh-issue-73991.CGknDf.rst @@ -0,0 +1,2 @@ +Add :meth:`pathlib.Path.copy`, which copies the content of one file to another, +like :func:`shutil.copyfile`. From 7fadfd82ebf6ea90b38cb3f2a046a51f8601a205 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 14 Jun 2024 20:25:35 +0300 Subject: [PATCH 09/88] gh-120361: Add `nonmember` test with enum flags inside to `test_enum` (GH-120364) * gh-120361: Add `nonmember` test with enum flags inside to `test_enum` --- Doc/library/enum.rst | 2 +- Lib/test/test_enum.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 8c604c2347a547..9cf94e342dad28 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -527,7 +527,7 @@ Data Types ``Flag`` is the same as :class:`Enum`, but its members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), ``^`` (*XOR*), and ``~`` (*INVERT*); - the results of those operators are members of the enumeration. + the results of those operations are (aliases of) members of the enumeration. .. method:: __contains__(self, value) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 529dfc62eff680..99fd16ba361e6f 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1495,6 +1495,27 @@ class SpamEnum(Enum): spam = nonmember(SpamEnumIsInner) self.assertTrue(SpamEnum.spam is SpamEnumIsInner) + def test_using_members_as_nonmember(self): + class Example(Flag): + A = 1 + B = 2 + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + + class Example(Flag): + A = auto() + B = auto() + ALL = nonmember(A | B) + + self.assertEqual(Example.A.value, 1) + self.assertEqual(Example.B.value, 2) + self.assertEqual(Example.ALL, 3) + self.assertIs(type(Example.ALL), int) + def test_nested_classes_in_enum_with_member(self): """Support locally-defined nested classes.""" class Outer(Enum): From ed60ab5fab6d187068cb3e0f0d4192ebf3a228b7 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 14 Jun 2024 11:25:23 -0700 Subject: [PATCH 10/88] gh-119824: Print stack entry when user input is needed (#119882) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/library/pdb.rst | 10 +++- Lib/pdb.py | 50 +++++++++++++++---- Lib/test/test_pdb.py | 50 ++++++++++++++++--- ...-05-31-21-17-43.gh-issue-119824.CQlxWV.rst | 1 + 4 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index f6085171dccb38..b1e9392ecfd927 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -321,11 +321,17 @@ can be overridden by the local file. argument must be an identifier, ``help exec`` must be entered to get help on the ``!`` command. -.. pdbcommand:: w(here) +.. pdbcommand:: w(here) [count] - Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) + Print a stack trace, with the most recent frame at the bottom. if *count* + is 0, print the current frame entry. If *count* is negative, print the least + recent - *count* frames. If *count* is positive, print the most recent + *count* frames. An arrow (``>``) indicates the current frame, which determines the context of most commands. + .. versionchanged:: 3.14 + *count* argument is added. + .. pdbcommand:: d(own) [count] Move the current frame *count* (default one) levels down in the stack trace diff --git a/Lib/pdb.py b/Lib/pdb.py index ba84a29aa2f669..ddbfb9d2bb6244 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -603,10 +603,18 @@ def interaction(self, frame, tb_or_exc): assert tb is not None, "main exception must have a traceback" with self._hold_exceptions(_chained_exceptions): self.setup(frame, tb) - # if we have more commands to process, do not show the stack entry - if not self.cmdqueue: + # We should print the stack entry if and only if the user input + # is expected, and we should print it right before the user input. + # If self.cmdqueue is not empty, we append a "w 0" command to the + # queue, which is equivalent to print_stack_entry + if self.cmdqueue: + self.cmdqueue.append('w 0') + else: self.print_stack_entry(self.stack[self.curindex]) self._cmdloop() + # If "w 0" is not used, pop it out + if self.cmdqueue and self.cmdqueue[-1] == 'w 0': + self.cmdqueue.pop() self.forget() def displayhook(self, obj): @@ -1401,16 +1409,24 @@ def do_clear(self, arg): complete_cl = _complete_location def do_where(self, arg): - """w(here) + """w(here) [count] - Print a stack trace, with the most recent frame at the bottom. + Print a stack trace. If count is not specified, print the full stack. + If count is 0, print the current frame entry. If count is positive, + print count entries from the most recent frame. If count is negative, + print -count entries from the least recent frame. An arrow indicates the "current frame", which determines the context of most commands. 'bt' is an alias for this command. """ - if arg: - self._print_invalid_arg(arg) - return - self.print_stack_trace() + if not arg: + count = None + else: + try: + count = int(arg) + except ValueError: + self.error('Invalid count (%s)' % arg) + return + self.print_stack_trace(count) do_w = do_where do_bt = do_where @@ -2065,10 +2081,22 @@ def complete_unalias(self, text, line, begidx, endidx): # It is also consistent with the up/down commands (which are # compatible with dbx and gdb: up moves towards 'main()' # and down moves towards the most recent stack frame). - - def print_stack_trace(self): + # * if count is None, prints the full stack + # * if count = 0, prints the current frame entry + # * if count < 0, prints -count least recent frame entries + # * if count > 0, prints count most recent frame entries + + def print_stack_trace(self, count=None): + if count is None: + stack_to_print = self.stack + elif count == 0: + stack_to_print = [self.stack[self.curindex]] + elif count < 0: + stack_to_print = self.stack[:-count] + else: + stack_to_print = self.stack[-count:] try: - for frame_lineno in self.stack: + for frame_lineno in stack_to_print: self.print_stack_entry(frame_lineno) except KeyboardInterrupt: pass diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index cf69bc415c9b69..5edf68dc3b429b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -781,7 +781,7 @@ def test_pdb_where_command(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() >>> def f(): - ... g(); + ... g() >>> def test_function(): ... f() @@ -789,8 +789,13 @@ def test_pdb_where_command(): >>> with PdbTestInput([ # doctest: +ELLIPSIS ... 'w', ... 'where', + ... 'w 1', + ... 'w invalid', ... 'u', ... 'w', + ... 'w 0', + ... 'w 100', + ... 'w -100', ... 'continue', ... ]): ... test_function() @@ -798,35 +803,63 @@ def test_pdb_where_command(): -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); + -> g() > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... - (8)() + (13)() -> test_function() (2)test_function() -> f() (2)f() - -> g(); + -> g() > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 1 + > (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w invalid + *** Invalid count (invalid) (Pdb) u > (2)f() - -> g(); + -> g() (Pdb) w ... - (8)() + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w 0 + > (2)f() + -> g() + (Pdb) w 100 + ... + (13)() -> test_function() (2)test_function() -> f() > (2)f() - -> g(); + -> g() + (2)g() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) w -100 + ... + (13)() + -> test_function() + (2)test_function() + -> f() + > (2)f() + -> g() (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue @@ -3179,6 +3212,7 @@ def test_pdbrc_basic(self): stdout, stderr = self.run_pdb_script(script, 'q\n', pdbrc=pdbrc, remove_home=True) self.assertNotIn("SyntaxError", stdout) self.assertIn("a+8=9", stdout) + self.assertIn("-> b = 2", stdout) def test_pdbrc_empty_line(self): """Test that empty lines in .pdbrc are ignored.""" diff --git a/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst b/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst new file mode 100644 index 00000000000000..fd6d8d79a9d157 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-31-21-17-43.gh-issue-119824.CQlxWV.rst @@ -0,0 +1 @@ +Print stack entry in :mod:`pdb` when and only when user input is needed. From 05df063ad80becc1ba6bd07d67b55b5965f32375 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Jun 2024 20:39:50 +0200 Subject: [PATCH 11/88] gh-120417: Fix "imported but unused" linter warnings (#120461) Add __all__ to the following modules: importlib.machinery, importlib.util and xml.sax. Add also "# noqa: F401" in collections.abc, subprocess and xml.sax. * Sort __all__; remove collections.abc.__all__; remove private names * Add tests --- Lib/collections/abc.py | 4 +-- Lib/importlib/machinery.py | 8 ++++++ Lib/importlib/util.py | 6 +++++ Lib/subprocess.py | 2 +- Lib/test/test_importlib/test_api.py | 40 +++++++++++++++++++++++++++++ Lib/test/test_sax.py | 18 ++++++++++++- Lib/xml/sax/__init__.py | 14 +++++++--- 7 files changed, 84 insertions(+), 8 deletions(-) diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 86ca8b8a8414b3..bff76291634604 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,3 @@ from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias +from _collections_abc import __all__ # noqa: F401 +from _collections_abc import _CallableGenericAlias # noqa: F401 diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index fbd30b159fb752..6e294d59bfdcb9 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -19,3 +19,11 @@ def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index c94a148e4c50e0..7243d052cc27f3 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -270,3 +270,9 @@ def exec_module(self, module): loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/subprocess.py b/Lib/subprocess.py index b2dcb1454c139e..bc08878db313df 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -79,7 +79,7 @@ if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 2a35f3dcb7210c..973237c0791a3e 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,6 +6,7 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper import types @@ -437,5 +438,44 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 9b3014a94a081e..0d0f86c145b499 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -16,6 +16,7 @@ from xml.sax.handler import (feature_namespaces, feature_external_ges, LexicalHandler) from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl +from xml import sax from io import BytesIO, StringIO import codecs import os.path @@ -25,7 +26,7 @@ from urllib.error import URLError import urllib.request from test.support import os_helper -from test.support import findfile +from test.support import findfile, check__all__ from test.support.os_helper import FakePath, TESTFN @@ -1557,5 +1558,20 @@ def characters(self, content): self.assertEqual(self.char_index, 2) +class TestModuleAll(unittest.TestCase): + def test_all(self): + extra = ( + 'ContentHandler', + 'ErrorHandler', + 'InputSource', + 'SAXException', + 'SAXNotRecognizedException', + 'SAXNotSupportedException', + 'SAXParseException', + 'SAXReaderNotAvailable', + ) + check__all__(self, sax, extra=extra) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index b657310207cfe5..fe4582c6f8b758 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -21,9 +21,9 @@ from .xmlreader import InputSource from .handler import ContentHandler, ErrorHandler -from ._exceptions import SAXException, SAXNotRecognizedException, \ - SAXParseException, SAXNotSupportedException, \ - SAXReaderNotAvailable +from ._exceptions import (SAXException, SAXNotRecognizedException, + SAXParseException, SAXNotSupportedException, + SAXReaderNotAvailable) def parse(source, handler, errorHandler=ErrorHandler()): @@ -55,7 +55,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): # tell modulefinder that importing sax potentially imports expatreader _false = 0 if _false: - import xml.sax.expatreader + import xml.sax.expatreader # noqa: F401 import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: @@ -92,3 +92,9 @@ def make_parser(parser_list=()): def _create_parser(parser_name): drv_module = __import__(parser_name,{},{},['create_parser']) return drv_module.create_parser() + + +__all__ = ['ContentHandler', 'ErrorHandler', 'InputSource', 'SAXException', + 'SAXNotRecognizedException', 'SAXNotSupportedException', + 'SAXParseException', 'SAXReaderNotAvailable', + 'default_parser_list', 'make_parser', 'parse', 'parseString'] From b2e71ff4f8fa5b7d8117dd8125137aee3d01f015 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 14 Jun 2024 15:29:09 -0400 Subject: [PATCH 12/88] gh-120161: Fix a Crash in the _datetime Module (gh-120182) In gh-120009 I used an atexit hook to finalize the _datetime module's static types at interpreter shutdown. However, atexit hooks are executed very early in finalization, which is a problem in the few cases where a subclass of one of those static types is still alive until the final GC collection. The static builtin types don't have this probably because they are finalized toward the end, after the final GC collection. To avoid the problem for _datetime, I have applied a similar approach here. Also, credit goes to @mgorny and @neonene for the new tests. FYI, I would have liked to take a slightly cleaner approach with managed static types, but wanted to get a smaller fix in first for the sake of backporting. I'll circle back to the cleaner approach with a future change on the main branch. --- Include/internal/pycore_typeobject.h | 24 ++++-- Lib/test/datetimetester.py | 44 +++++++++- ...-06-06-17-24-43.gh-issue-120161.DahNXV.rst | 2 + Modules/_datetimemodule.c | 48 +---------- Objects/typeobject.c | 85 +++++++++++++++---- Python/pylifecycle.c | 1 + 6 files changed, 133 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index bc295b1b066bd1..32bd19d968b917 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -17,11 +17,25 @@ extern "C" { #define _Py_TYPE_BASE_VERSION_TAG (2<<16) #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) +/* For now we hard-code this to a value for which we are confident + all the static builtin types will fit (for all builds). */ +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 +#define _Py_MAX_MANAGED_STATIC_TYPES \ + (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) + struct _types_runtime_state { /* Used to set PyTypeObject.tp_version_tag for core static types. */ // bpo-42745: next_version_tag remains shared by all interpreters // because of static types. unsigned int next_version_tag; + + struct { + struct { + PyTypeObject *type; + int64_t interp_count; + } types[_Py_MAX_MANAGED_STATIC_TYPES]; + } managed_static; }; @@ -42,11 +56,6 @@ struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; }; -/* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 -#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 - typedef struct { PyTypeObject *type; int isbuiltin; @@ -133,6 +142,7 @@ struct types_state { extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); +extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); extern void _PyTypes_AfterFork(void); @@ -171,10 +181,6 @@ extern managed_static_type_state * _PyStaticType_GetState( PyAPI_FUNC(int) _PyStaticType_InitForExtension( PyInterpreterState *interp, PyTypeObject *self); -PyAPI_FUNC(void) _PyStaticType_FiniForExtension( - PyInterpreterState *interp, - PyTypeObject *self, - int final); /* Like PyType_GetModuleState, but skips verification diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 45188731eed688..70e2e2cccdc55f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -23,7 +23,7 @@ from test import support from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST -from test.support import warnings_helper +from test.support import script_helper, warnings_helper import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -6822,6 +6822,48 @@ def run(type_checker, obj): self.assertEqual(ret, 0) +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) return standard_tests diff --git a/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst b/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst new file mode 100644 index 00000000000000..c378cac44c97bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-06-17-24-43.gh-issue-120161.DahNXV.rst @@ -0,0 +1,2 @@ +:mod:`datetime` no longer crashes in certain complex reference cycle +situations. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cb4622893375d7..5c4f1f888d17ee 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7129,37 +7129,6 @@ clear_state(datetime_state *st) } -/* --------------------------------------------------------------------------- - * Global module state. - */ - -// If we make _PyStaticType_*ForExtension() public -// then all this should be managed by the runtime. - -static struct { - PyMutex mutex; - int64_t interp_count; -} _globals = {0}; - -static void -callback_for_interp_exit(void *Py_UNUSED(data)) -{ - PyInterpreterState *interp = PyInterpreterState_Get(); - - assert(_globals.interp_count > 0); - PyMutex_Lock(&_globals.mutex); - _globals.interp_count -= 1; - int final = !_globals.interp_count; - PyMutex_Unlock(&_globals.mutex); - - /* They must be done in reverse order so subclasses are finalized - * before base classes. */ - for (size_t i = Py_ARRAY_LENGTH(capi_types); i > 0; i--) { - PyTypeObject *type = capi_types[i-1]; - _PyStaticType_FiniForExtension(interp, type, final); - } -} - static int init_static_types(PyInterpreterState *interp, int reloading) { @@ -7182,19 +7151,6 @@ init_static_types(PyInterpreterState *interp, int reloading) } } - PyMutex_Lock(&_globals.mutex); - assert(_globals.interp_count >= 0); - _globals.interp_count += 1; - PyMutex_Unlock(&_globals.mutex); - - /* It could make sense to add a separate callback - * for each of the types. However, for now we can take the simpler - * approach of a single callback. */ - if (PyUnstable_AtExit(interp, callback_for_interp_exit, NULL) < 0) { - callback_for_interp_exit(NULL); - return -1; - } - return 0; } @@ -7379,8 +7335,8 @@ module_clear(PyObject *mod) PyInterpreterState *interp = PyInterpreterState_Get(); clear_current_module(interp, mod); - // We take care of the static types via an interpreter atexit hook. - // See callback_for_interp_exit() above. + // The runtime takes care of the static types for us. + // See _PyTypes_FiniExtTypes().. return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8ecab555454cdc..98e00bd25c3205 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -159,18 +159,28 @@ managed_static_type_index_clear(PyTypeObject *self) self->tp_subclasses = NULL; } -static inline managed_static_type_state * -static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) +static PyTypeObject * +static_ext_type_lookup(PyInterpreterState *interp, size_t index, + int64_t *p_interp_count) { - return &(interp->types.builtins.initialized[ - managed_static_type_index_get(self)]); -} + assert(interp->runtime == &_PyRuntime); + assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); -static inline managed_static_type_state * -static_ext_type_state_get(PyInterpreterState *interp, PyTypeObject *self) -{ - return &(interp->types.for_extensions.initialized[ - managed_static_type_index_get(self)]); + size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + int64_t interp_count = + _PyRuntime.types.managed_static.types[full_index].interp_count; + assert((interp_count == 0) == + (_PyRuntime.types.managed_static.types[full_index].type == NULL)); + *p_interp_count = interp_count; + + PyTypeObject *type = interp->types.for_extensions.initialized[index].type; + if (type == NULL) { + return NULL; + } + assert(!interp->types.for_extensions.initialized[index].isbuiltin); + assert(type == _PyRuntime.types.managed_static.types[full_index].type); + assert(managed_static_type_index_is_set(type)); + return type; } static managed_static_type_state * @@ -202,6 +212,8 @@ static void managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, int isbuiltin, int initial) { + assert(interp->runtime == &_PyRuntime); + size_t index; if (initial) { assert(!managed_static_type_index_is_set(self)); @@ -228,6 +240,21 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); } } + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + + assert((initial == 1) == + (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + _PyRuntime.types.managed_static.types[full_index].interp_count += 1; + + if (initial) { + assert(_PyRuntime.types.managed_static.types[full_index].type == NULL); + _PyRuntime.types.managed_static.types[full_index].type = self; + } + else { + assert(_PyRuntime.types.managed_static.types[full_index].type == self); + } managed_static_type_state *state = isbuiltin ? &(interp->types.builtins.initialized[index]) @@ -256,15 +283,28 @@ static void managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, int isbuiltin, int final) { + size_t index = managed_static_type_index_get(self); + size_t full_index = isbuiltin + ? index + : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + managed_static_type_state *state = isbuiltin - ? static_builtin_state_get(interp, self) - : static_ext_type_state_get(interp, self); + ? &(interp->types.builtins.initialized[index]) + : &(interp->types.for_extensions.initialized[index]); + assert(state != NULL); + + assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. + _PyRuntime.types.managed_static.types[full_index].interp_count -= 1; if (final) { + assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + _PyRuntime.types.managed_static.types[full_index].type = NULL; + managed_static_type_index_clear(self); } @@ -840,8 +880,12 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); + // All the managed static types should have been finalized already. + assert(interp->types.for_extensions.num_initialized == 0); + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + assert(interp->types.for_extensions.initialized[i].type == NULL); + } assert(interp->types.builtins.num_initialized == 0); - // All the static builtin types should have been finalized already. for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { assert(interp->types.builtins.initialized[i].type == NULL); } @@ -5834,9 +5878,20 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, } void -_PyStaticType_FiniForExtension(PyInterpreterState *interp, PyTypeObject *type, int final) +_PyTypes_FiniExtTypes(PyInterpreterState *interp) { - fini_static_type(interp, type, 0, final); + for (size_t i = _Py_MAX_MANAGED_STATIC_EXT_TYPES; i > 0; i--) { + if (interp->types.for_extensions.num_initialized == 0) { + break; + } + int64_t count = 0; + PyTypeObject *type = static_ext_type_lookup(interp, i-1, &count); + if (type == NULL) { + continue; + } + int final = (count == 1); + fini_static_type(interp, type, 0, final); + } } void diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index cbdf5c1b771fff..3639cf6712053e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1818,6 +1818,7 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyTypes_FiniExtTypes(interp); _PyUnicode_FiniTypes(interp); _PySys_FiniTypes(interp); _PyXI_FiniTypes(interp); From e3b6cff33122554de0ef598664f5cd98de4fed6b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 14 Jun 2024 18:12:35 -0400 Subject: [PATCH 13/88] gh-120524: Temporarily Skip test_create_many_threaded In test_interpreters.test_stress (gh-120525) --- Lib/test/test_interpreters/test_stress.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index e400535b2a0e4e..40d2d77a7b9d3e 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -22,6 +22,7 @@ def test_create_many_sequential(self): interp = interpreters.create() alive.append(interp) + @unittest.skip('(temporary) gh-120524: there is a race that needs fixing') @support.requires_resource('cpu') def test_create_many_threaded(self): alive = [] From 92f6d400f76b6a04dddd944568870f689c8fab5f Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 15 Jun 2024 08:05:18 +0800 Subject: [PATCH 14/88] gh-119819: Conditional skip of logging tests that require multiprocessing subprocess support (#120476) Skip tests that require multiprocessing subprocess support. --- Lib/test/test_logging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index ef2d4a621be962..504862ad53395e 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3898,6 +3898,7 @@ def do_queuehandler_configuration(self, qspec, lspec): self.addCleanup(os.remove, fn) @threading_helper.requires_working_threading() + @support.requires_subprocess() def test_config_queue_handler(self): q = CustomQueue() dq = { @@ -3926,12 +3927,10 @@ def test_config_queue_handler(self): msg = str(ctx.exception) self.assertEqual(msg, "Unable to configure handler 'ah'") + @support.requires_subprocess() def test_multiprocessing_queues(self): # See gh-119819 - # will skip test if it's not available - import_helper.import_module('_multiprocessing') - cd = copy.deepcopy(self.config_queue_handler) from multiprocessing import Queue as MQ, Manager as MM q1 = MQ() # this can't be pickled From 5c58e728b1391c258b224fc6d88f62f42c725026 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Sat, 15 Jun 2024 08:05:30 +0800 Subject: [PATCH 15/88] gh-117398: Use the correct module loader for iOS in datetime CAPI test. (#120477) Use the correct loader for iOS. --- Lib/test/datetimetester.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 70e2e2cccdc55f..e55b738eb4a975 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -6786,6 +6786,13 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" if {_interpreters is None}: import _testcapi as module @@ -6795,7 +6802,7 @@ def test_type_check_in_subinterp(self): import importlib.util fullname = '_testcapi_datetime' origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{extension_loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) From d4039d3f6f8cb7738c5cd272dde04171446dfd2b Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Fri, 14 Jun 2024 22:33:09 -0700 Subject: [PATCH 16/88] gh-120526: Correct signature of map() builtin (GH-120528) map() requires at least one iterable arg. Signed-off-by: Adam Williamson --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c4d3ecbeeff0e6..6e50623cafa4ed 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1475,7 +1475,7 @@ static PyMethodDef map_methods[] = { PyDoc_STRVAR(map_doc, -"map(function, /, *iterables)\n\ +"map(function, iterable, /, *iterables)\n\ --\n\ \n\ Make an iterator that computes the function using arguments from\n\ From 42ebdd83bb194f054fe5a10b3caa0c3a95be3679 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 15 Jun 2024 13:33:14 +0300 Subject: [PATCH 17/88] gh-120544: Add `else: fail()` to tests where exception is expected (#120545) --- Lib/test/test_exceptions.py | 2 ++ Lib/test/test_unittest/test_case.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 9460d1f1c864b9..e4f2e3a97b8bb8 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1859,6 +1859,8 @@ def f(): except self.failureException: with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) + else: + self.fail("assertRaisesRegex should have failed.") self.assertIn("aab", err.getvalue()) diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index 17420909402107..b4b2194a09cf9f 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1151,6 +1151,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1167,6 +1169,8 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualwithEmptyString(self): '''Verify when there is an empty string involved, the diff output @@ -1184,6 +1188,8 @@ def testAssertEqualwithEmptyString(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMissingNewlineTerminator(self): '''Verifying format of diff output from assertEqual involving strings @@ -1204,6 +1210,8 @@ def testAssertEqualMultipleLinesMissingNewlineTerminator(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): '''Verifying format of diff output from assertEqual involving strings @@ -1227,6 +1235,8 @@ def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: From c501261c919ceb97c850ef9427a93326f06a8f2e Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Sat, 15 Jun 2024 19:04:14 +0800 Subject: [PATCH 18/88] gh-120495: Fix incorrect exception handling in Tab Nanny (#120498) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/tabnanny.py | 8 ++++---- Lib/test/test_tabnanny.py | 2 +- Misc/ACKS | 1 + .../2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 7e56d4a48d1d00..c0097351b269f2 100644 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -105,14 +105,14 @@ def check(file): errprint("%r: Token Error: %s" % (file, msg)) return - except SyntaxError as msg: - errprint("%r: Token Error: %s" % (file, msg)) - return - except IndentationError as msg: errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index cc122cafc7985c..30dcb3e3c4f4f9 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: - stderr = f"{file_path!r}: Token Error: " + stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') self.validate_cmd(file_path, stderr=stderr, expect_failure=True) diff --git a/Misc/ACKS b/Misc/ACKS index 2f4c0793437fb6..a406fca8744a5f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1099,6 +1099,7 @@ Ivan Levkivskyi Ben Lewis William Lewis Akira Li +Jiahao Li Robert Li Xuanji Li Zekun Li diff --git a/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst b/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst new file mode 100644 index 00000000000000..d5114c3d3c904c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-14-20-05-25.gh-issue-120495.OxgZKB.rst @@ -0,0 +1 @@ +Fix incorrect exception handling in Tab Nanny. Patch by Wulian233. From 99d62f902e43c08ebec5a292fd3b30a9fc4cba69 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 15 Jun 2024 13:51:58 +0100 Subject: [PATCH 19/88] Add some more edge-case tests for `inspect.get_annotations` with `eval_str=True` (#120550) --- .../inspect_stringized_annotations_pep695.py | 23 ++++++++++++++---- Lib/test/test_inspect/test_inspect.py | 24 +++++++++++++------ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py index 723822f8eaa92d..39bfe2edb03f30 100644 --- a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -45,6 +45,13 @@ def generic_method[Foo, **Bar]( def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + def nested(): from types import SimpleNamespace from inspect import get_annotations @@ -53,7 +60,7 @@ def nested(): Spam = memoryview - class E[Eggs, **Spam]: + class F[Eggs, **Spam]: x: Eggs y: Spam @@ -63,10 +70,18 @@ def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + return SimpleNamespace( - E=E, - E_annotations=get_annotations(E, eval_str=True), - E_meth_annotations=get_annotations(E.generic_method, eval_str=True), + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), generic_func=generic_function, generic_func_annotations=get_annotations(generic_function, eval_str=True) ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 140efac530afb2..ea8735d8f06459 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1770,26 +1770,36 @@ def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_v ) ) + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): + self.assertEqual( + inspect.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + def test_pep_695_generics_with_future_annotations_nested_in_function(self): results = inspect_stringized_annotations_pep695.nested() self.assertEqual( - set(results.E_annotations.values()), - set(results.E.__type_params__) + set(results.F_annotations.values()), + set(results.F.__type_params__) ) self.assertEqual( - set(results.E_meth_annotations.values()), - set(results.E.generic_method.__type_params__) + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__) ) self.assertNotEqual( - set(results.E_meth_annotations.values()), - set(results.E.__type_params__) + set(results.F_meth_annotations.values()), + set(results.F.__type_params__) ) self.assertEqual( - set(results.E_meth_annotations.values()).intersection(results.E.__type_params__), + set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), set() ) + self.assertEqual(results.G_annotations, {"x": str}) + self.assertEqual( set(results.generic_func_annotations.values()), set(results.generic_func.__type_params__) From 6f63dfff6f493b405f3422210a168369e1e7a35d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 15 Jun 2024 22:39:22 +0800 Subject: [PATCH 20/88] gh-117657: Make PyType_HasFeature (exported version) atomic (#120484) Make PyType_HasFeature (exported version) atomic --- Include/object.h | 6 +++++- Objects/typeobject.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Include/object.h b/Include/object.h index 4a39ada8c7daa4..f71aaee7efe6ee 100644 --- a/Include/object.h +++ b/Include/object.h @@ -756,7 +756,11 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) // PyTypeObject is opaque in the limited C API flags = PyType_GetFlags(type); #else - flags = type->tp_flags; +# ifdef Py_GIL_DISABLED + flags = _Py_atomic_load_ulong_relaxed(&type->tp_flags); +# else + flags = type->tp_flags; +# endif #endif return ((flags & feature) != 0); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 98e00bd25c3205..eb296414bb7bef 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3599,7 +3599,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds) unsigned long PyType_GetFlags(PyTypeObject *type) { - return type->tp_flags; + return FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags); } From 9e0b11eb21930b7b8e4a396200a921e9985cfca4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 15 Jun 2024 08:18:16 -0700 Subject: [PATCH 21/88] annotations: expand documentation on "simple" assignment targets (#120535) This behavior is rather surprising and it was not clearly specified. Co-authored-by: Alex Waygood --- Doc/library/ast.rst | 10 +++++++--- Doc/reference/simple_stmts.rst | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 9ee56b92431b57..f7e8afa7000392 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -891,9 +891,13 @@ Statements An assignment with a type annotation. ``target`` is a single node and can be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. + node. ``value`` is a single optional node. + + ``simple`` is always either 0 (indicating a "complex" target) or 1 + (indicating a "simple" target). A "simple" target consists solely of a + :class:`Name` node that does not appear between parentheses; all other + targets are considered complex. Only simple targets appear in + the :attr:`__annotations__` dictionary of modules and classes. .. doctest:: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a253482156d3b4..4f6c0c63ae42be 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -333,7 +333,9 @@ statement, of a variable or attribute annotation and an optional assignment stat The difference from normal :ref:`assignment` is that only a single target is allowed. -For simple names as assignment targets, if in class or module scope, +The assignment target is considered "simple" if it consists of a single +name that is not enclosed in parentheses. +For simple assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module attribute :attr:`__annotations__` that is a dictionary mapping from variable names (mangled if private) to @@ -341,7 +343,8 @@ evaluated annotations. This attribute is writable and is automatically created at the start of class or module body execution, if annotations are found statically. -For expressions as assignment targets, the annotations are evaluated if +If the assignment target is not simple (an attribute, subscript node, or +parenthesized name), the annotation is evaluated if in class or module scope, but not stored. If a name is annotated in a function scope, then this name is local for From 31d1d72d7e24e0427df70f7dd14b9baff28a4f89 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 15 Jun 2024 20:56:40 +0300 Subject: [PATCH 22/88] gh-120541: Improve the "less" prompt in pydoc (GH-120543) When help() is called with non-string argument, use __qualname__ or __name__ if available, otherwise use "{typename} object". --- Lib/pydoc.py | 9 ++- Lib/test/test_pydoc/test_pydoc.py | 62 +++++++++++++++---- ...-06-15-12-04-46.gh-issue-120541.d3cc5y.rst | 2 + 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index be5cd9a80db710..768c3dcb11ec59 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1755,7 +1755,14 @@ def doc(thing, title='Python Library Documentation: %s', forceload=0, """Display text documentation, given an object or a path to an object.""" if output is None: try: - what = thing if isinstance(thing, str) else type(thing).__name__ + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' pager(render_doc(thing, title, forceload), f'Help on {what!s}') except ImportError as exc: if is_cli: diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index a17c16cc73cf0e..b520cfd0b50e38 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -31,7 +31,7 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, spawn_python) from test.support import threading_helper -from test.support import (reap_children, captured_output, captured_stdout, +from test.support import (reap_children, captured_stdout, captured_stderr, is_emscripten, is_wasi, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) @@ -680,9 +680,8 @@ def test_help_output_redirect(self, pager_mock): help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - with captured_output('stdout') as output, \ - captured_output('stderr') as err, \ - StringIO() as buf: + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() helper = pydoc.Helper(output=buf) helper.help(module) result = buf.getvalue().strip() @@ -706,9 +705,8 @@ def test_help_output_redirect_various_requests(self, pager_mock): def run_pydoc_for_request(request, expected_text_part): """Helper function to run pydoc with its output redirected""" - with captured_output('stdout') as output, \ - captured_output('stderr') as err, \ - StringIO() as buf: + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() helper = pydoc.Helper(output=buf) helper.help(request) result = buf.getvalue().strip() @@ -742,6 +740,45 @@ def run_pydoc_for_request(request, expected_text_part): run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') # test for pydoc.Helper() instance skipped because it is always meant to be interactive + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) + self.assertEqual('', err.getvalue()) + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', 'Help on method_descriptor in str:') + run_pydoc_pager(str.upper, 'str.upper', 'Help on method_descriptor:') + run_pydoc_pager(str.__add__, 'str.__add__', 'Help on wrapper_descriptor:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + def test_showtopic(self): with captured_stdout() as showtopic_io: helper = pydoc.Helper() @@ -775,9 +812,8 @@ def test_showtopic_output_redirect(self, pager_mock): # Helper.showtopic should be redirected self.maxDiff = None - with captured_output('stdout') as output, \ - captured_output('stderr') as err, \ - StringIO() as buf: + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() helper = pydoc.Helper(output=buf) helper.showtopic('with') result = buf.getvalue().strip() @@ -790,7 +826,7 @@ def test_showtopic_output_redirect(self, pager_mock): def test_lambda_with_return_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"return": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a, b, c) -> int", helptext) @@ -798,7 +834,7 @@ def test_lambda_with_return_annotation(self): def test_lambda_without_return_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"a": int, "b": int, "c": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a: int, b: int, c: int)", helptext) @@ -806,7 +842,7 @@ def test_lambda_without_return_annotation(self): def test_lambda_with_return_and_params_annotation(self): func = lambda a, b, c: 1 func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} - with captured_output('stdout') as help_io: + with captured_stdout() as help_io: pydoc.help(func) helptext = help_io.getvalue() self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) diff --git a/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst b/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst new file mode 100644 index 00000000000000..bf8830c6c50386 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-15-12-04-46.gh-issue-120541.d3cc5y.rst @@ -0,0 +1,2 @@ +Improve the prompt in the "less" pager when :func:`help` is called with +non-string argument. From 08d09cf5ba041c9c5c3860200b56bab66fd44a23 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Sat, 15 Jun 2024 20:46:39 +0200 Subject: [PATCH 23/88] gh-112346: Always set OS byte to 255, simpler gzip.compress function. (GH-120486) This matches the output behavior in 3.10 and earlier; the optimization in 3.11 allowed the zlib library's "os" value to be filled in instead in the circumstance when mtime was 0. this keeps things consistent. --- Doc/library/gzip.rst | 8 ++-- Lib/gzip.py | 38 ++++--------------- Lib/test/test_gzip.py | 12 +++++- ...4-06-12-10-00-31.gh-issue-90425.5CfkKG.rst | 2 + 4 files changed, 26 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-12-10-00-31.gh-issue-90425.5CfkKG.rst diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 965da5981f6dbc..152cba4f653cb4 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -188,9 +188,7 @@ The module defines the following items: Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this - function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. - The zlib function is faster. + the :class:`GzipFile` constructor above. .. versionadded:: 3.2 .. versionchanged:: 3.8 @@ -200,6 +198,10 @@ The module defines the following items: streamed fashion. Calls with *mtime* set to ``0`` are delegated to :func:`zlib.compress` for better speed. + .. versionchanged:: 3.13 + The gzip header OS byte is guaranteed to be set to 255 when this function + is used as was the case in 3.10 and earlier. + .. function:: decompress(data) Decompress the *data*, returning a :class:`bytes` object containing the diff --git a/Lib/gzip.py b/Lib/gzip.py index 0d19c84c59cfa7..ba753ce3050dd8 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -580,27 +580,6 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" Date: Sun, 16 Jun 2024 13:36:10 +0800 Subject: [PATCH 24/88] gh-120572: add missing parentheses in TypeIs documentation (#120573) --- Doc/library/typing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 94de64fcf835fc..bf0ff9bd348553 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1454,8 +1454,8 @@ These can be used as types in annotations. They all support subscription using to write such functions in a type-safe manner. If a ``TypeIs`` function is a class or instance method, then the type in - ``TypeIs`` maps to the type of the second parameter after ``cls`` or - ``self``. + ``TypeIs`` maps to the type of the second parameter (after ``cls`` or + ``self``). In short, the form ``def foo(arg: TypeA) -> TypeIs[TypeB]: ...``, means that if ``foo(arg)`` returns ``True``, then ``arg`` is an instance From cf49ef78f894e418bea7de23dde9b01d6235889d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 16 Jun 2024 01:55:47 -0400 Subject: [PATCH 25/88] gh-120360: Add self as IDLE doc owner (#120571) Add self as IDLE doc owner --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1f9047ab97e934..eb7cc88565f6d0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -212,6 +212,7 @@ Doc/c-api/stable.rst @encukou **/*ensurepip* @pfmoore @pradyunsg **/*idlelib* @terryjreedy +/Doc/library/idle.rst @terryjreedy **/*typing* @JelleZijlstra @AlexWaygood From 0c0348adbfca991f78b3aaa6790e5c26606a1c0f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 16 Jun 2024 11:26:13 +0300 Subject: [PATCH 26/88] gh-120579: Guard `_testcapi` import in `test_free_threading` (#120580) --- Lib/test/test_free_threading/test_dict.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index f877582e6b565c..3126458e08e50a 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -8,7 +8,10 @@ from threading import Thread from unittest import TestCase -from _testcapi import dict_version +try: + import _testcapi +except ImportError: + _testcapi = None from test.support import threading_helper @@ -139,7 +142,9 @@ def writer_func(l): for ref in thread_list: self.assertIsNone(ref()) + @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_dict_version(self): + dict_version = _testcapi.dict_version THREAD_COUNT = 10 DICT_COUNT = 10000 lists = [] From 192d17c3fd9945104bc0303cf248bb0d074d260e Mon Sep 17 00:00:00 2001 From: Idan Kapustian <71190257+idankap@users.noreply.github.com> Date: Sun, 16 Jun 2024 15:15:03 +0300 Subject: [PATCH 27/88] gh-120485: Add an override of `allow_reuse_port` on classes subclassing `socketserver.TCPServer` (GH-120488) Co-authored-by: Vinay Sajip --- Lib/http/server.py | 3 ++- Lib/logging/config.py | 3 ++- Lib/test/test_logging.py | 1 + Lib/xmlrpc/server.py | 1 + .../2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 7d0da5052d2d4d..2d010649e56b51 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -129,7 +129,8 @@ class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = True def server_bind(self): """Override server_bind to store the server name.""" diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 9de84e527b18ac..d2f23e53f35c57 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -984,7 +984,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): A simple TCP socket-based logging config receiver. """ - allow_reuse_address = 1 + allow_reuse_address = True + allow_reuse_port = True def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 504862ad53395e..5192ce252a4d4c 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1038,6 +1038,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True + allow_reuse_port = True def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 4dddb1d10e08bd..90a356fbb8eae4 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -578,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = True # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst new file mode 100644 index 00000000000000..f41c233908362f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-14-07-52-00.gh-issue-120485.yy4K4b.rst @@ -0,0 +1 @@ +Add an override of ``allow_reuse_port`` on classes subclassing ``socketserver.TCPServer`` where ``allow_reuse_address`` is also overridden. From b8484c6ad7fd14ca464e584b79821b4b906dd77a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 16 Jun 2024 06:51:17 -0600 Subject: [PATCH 28/88] Docs: remove temporary hardcoded links (#120348) --- Doc/tools/static/rtd_switcher.js | 35 +------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js index a67bb85505a9ca..f5dc7045a0dbc4 100644 --- a/Doc/tools/static/rtd_switcher.js +++ b/Doc/tools/static/rtd_switcher.js @@ -6,42 +6,9 @@ document.addEventListener("readthedocs-addons-data-ready", function(event) { const config = event.detail.data() - - // Add some mocked hardcoded versions pointing to the official - // documentation while migrating to Read the Docs. - // These are only for testing purposes. - // TODO: remove them when managing all the versions on Read the Docs, - // since all the "active, built and not hidden" versions will be shown automatically. - let versions = config.versions.active.concat([ - { - slug: "dev (3.14)", - urls: { - documentation: "https://docs.python.org/3.14/", - } - }, - { - slug: "dev (3.13)", - urls: { - documentation: "https://docs.python.org/3.13/", - } - }, - { - slug: "3.12", - urls: { - documentation: "https://docs.python.org/3.12/", - } - }, - { - slug: "3.11", - urls: { - documentation: "https://docs.python.org/3.11/", - } - }, - ]); - const versionSelect = `