From 6de6191a0c56a9b0d31ac1c32778081a9e2b0311 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 25 Apr 2023 14:03:24 -0700 Subject: [PATCH 01/78] Use `std::hash`, `std::equal_to` everywhere **except when libc++ is in use** (#4319) * Try using `std::hash`, `std::equal_to` everywhere. From PR #4316 we know that types in the unnamed namespace in different translation units do not compare equal, as desired. But do types in named namespaces compare equal, as desired? * Revert "Try using `std::hash`, `std::equal_to` everywhere." This reverts commit a06949a9265014b3c581396c4f37c45ccc03dea6. * Use "our own name-based hash and equality functions" for `std::type_index` only under macOS, based on results shown under https://github.com/pybind/pybind11/pull/4316#issuecomment-1305097879 * Patch in PR #4313: Minimal reproducer for clash when binding types defined in the unnamed namespace. * test_unnamed_namespace_b xfail for clang * `PYBIND11_INTERNALS_VERSION 5` * Add a note to docs/classes.rst * For compatibility with Google-internal testing, test_unnamed_namespace_a & test_unnamed_namespace_b need to work when imported in any order. * Trying "__GLIBCXX__ or Windows", based on observations from Google-internal testing. * Try _LIBCPP_VERSION * Account for libc++ behavior in tests and documentation. * Adjust expectations for Windows Clang (and make code less redundant). * Add WindowsClang to ci.yml Added block transferred from PR #4321 * Add clang-latest to name that appears in the GitHub Actions web view. * Tweak the note in classes.rst again. * Add `pip install --upgrade pip`, Show env, cosmetic changes Already tested under PR #4321 * Add macos_brew_install_llvm to ci.yml Added block transferred from PR #4324 * `test_cross_module_exception_translator` xfail 'Homebrew Clang' * Revert back to base version of .github/workflows/ci.yml (the ci.yml changes were merged under #4323 and #4326) * Fixes for ruff * Make updated condition in internals.h dependent on ABI version. * Remove PYBIND11_TEST_OVERRIDE when testing with PYBIND11_INTERNALS_VERSION=10000000 * Selectively exercise cmake `-DPYBIND11_TEST_OVERRIDE`: ubuntu, macos, windows Extra work added to quick jobs, based on timings below, to not increase the GHA start-to-last-job-finished time. ``` Duration ^ Number of pytest runs ^ ^ Job identifier ^ ^ ^ 0:03:48.024227 1 1___3___Clang_3.6___C++11___x64.txt 0:03:58.992814 1 2___3___Clang_3.7___C++11___x64.txt 0:04:25.758942 1 1___3.7___Debian___x86____Install.txt 0:04:50.148276 1 4___3___Clang_7___C++11___x64.txt 0:04:55.784558 1 13___3___Clang_15___C++20___x64.txt 0:04:57.048754 1 6___3___Clang_dev___C++11___x64.txt 0:05:00.485181 1 7___3___Clang_5___C++14___x64.txt 0:05:03.744964 1 2___3___almalinux8___x64.txt 0:05:06.222752 1 5___3___Clang_9___C++11___x64.txt 0:05:11.767022 1 2___3___GCC_7___C++17__x64.txt 0:05:18.634930 1 2___3.11__deadsnakes____x64.txt 0:05:22.810995 1 1___3___GCC_7___C++11__x64.txt 0:05:25.275317 1 12___3___Clang_14___C++20___x64.txt 0:05:32.058174 1 5___3___GCC_10___C++17__x64.txt 0:05:39.381351 1 7___3___GCC_12___C++20__x64.txt 0:05:40.502252 1 8___3___Clang_10___C++17___x64.txt 0:05:59.344905 1 3___3___Clang_3.9___C++11___x64.txt 0:06:10.825147 1 6___3___GCC_11___C++20__x64.txt 0:06:20.655443 1 3___3___almalinux9___x64.txt 0:06:22.472061 1 3___3___GCC_8___C++14__x64.txt 0:06:42.647406 1 11___3___Clang_13___C++20___x64.txt 0:06:53.352720 1 1___3.10___CUDA_11.7___Ubuntu_22.04.txt 0:07:07.357801 1 2___3.7___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=14.txt 0:07:09.057603 1 1___3___centos7___x64.txt 0:07:15.546282 1 1___3.8___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=17.txt 0:07:22.566022 1 4___3___GCC_8___C++17__x64.txt 0:08:13.592674 1 2___3.9___MSVC_2019__Debug____x86_-DCMAKE_CXX_STANDARD=20.txt 0:08:16.422768 1 9___3___Clang_11___C++20___x64.txt 0:08:21.168457 1 3___3.8___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=17.txt 0:08:27.129468 1 10___3___Clang_12___C++20___x64.txt 0:09:35.045470 1 1___3.10___windows-latest___clang-latest.txt 0:09:57.361843 1 1___3.9___MSVC_2022_C++20___x64.txt 0:10:35.187767 1 1___3.6___MSVC_2019___x86.txt 0:11:14.691200 4 2___3.9___ubuntu-20.04___x64.txt 0:11:37.701167 1 1_macos-latest___brew_install_llvm.txt 0:11:38.688299 4 4___3.11___ubuntu-20.04___x64.txt 0:11:52.720216 1 4___3.9___MSVC_2019___x86_-DCMAKE_CXX_STANDARD=20.txt 0:13:23.456591 4 6___pypy-3.8___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:13:25.863592 2 1___3___ICC_latest___x64.txt 0:13:32.411758 3 9___3.9___windows-2022___x64.txt 0:13:45.473377 4 3___3.10___ubuntu-20.04___x64.txt 0:13:55.366447 4 5___pypy-3.7___ubuntu-20.04___x64.txt 0:13:57.969502 3 10___3.10___windows-2022___x64.txt 0:14:19.837475 3 11___3.11___windows-2022___x64.txt 0:14:33.316770 4 1___3.6___ubuntu-20.04___x64_-DPYBIND11_FINDPYTHON=ON_-DCMA.txt 0:15:34.449278 4 22___3.6___windows-2019___x64_-DPYBIND11_FINDPYTHON=ON.txt 0:16:25.189055 2 1___3.9-dbg__deadsnakes____Valgrind___x64.txt 0:17:20.956667 4 15___3.6___macos-latest___x64.txt 0:17:27.513891 4 23___3.9___windows-2019___x64.txt 0:17:58.783286 3 8___3.6___windows-2022___x64.txt 0:18:25.917828 4 7___pypy-3.9___ubuntu-20.04___x64.txt 0:19:17.399820 3 13___pypy-3.8___windows-2022___x64.txt 0:19:45.002122 3 12___pypy-3.7___windows-2022___x64.txt 0:20:03.201926 4 16___3.9___macos-latest___x64.txt 0:20:15.415178 4 17___3.10___macos-latest___x64.txt 0:20:20.263216 4 20___pypy-3.8___macos-latest___x64.txt 0:20:31.998226 3 1___3___windows-latest___mingw64.txt 0:20:40.812286 4 18___3.11___macos-latest___x64.txt 0:22:47.714749 4 19___pypy-3.7___macos-latest___x64.txt 0:23:04.435859 3 2___3___windows-latest___mingw32.txt 0:25:48.719597 3 14___pypy-3.9___windows-2022___x64.txt 0:26:01.211688 4 21___pypy-3.9___macos-latest___x64.txt 0:28:19.971015 1 1___3___CentOS7__PGI_22.9___x64.txt ``` * Update skipif for Python 3.12a7 (the WIP needs to be handled in a separate PR). --- .github/workflows/upstream.yml | 1 - docs/classes.rst | 10 ++++++++ include/pybind11/detail/internals.h | 3 ++- tests/CMakeLists.txt | 2 ++ tests/test_exceptions.py | 2 +- tests/test_unnamed_namespace_a.cpp | 38 +++++++++++++++++++++++++++++ tests/test_unnamed_namespace_a.py | 34 ++++++++++++++++++++++++++ tests/test_unnamed_namespace_b.cpp | 13 ++++++++++ tests/test_unnamed_namespace_b.py | 5 ++++ 9 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/test_unnamed_namespace_a.cpp create mode 100644 tests/test_unnamed_namespace_a.py create mode 100644 tests/test_unnamed_namespace_b.cpp create mode 100644 tests/test_unnamed_namespace_b.py diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index be3cd405..be643ddf 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -95,7 +95,6 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" - name: Build (unstable ABI) run: cmake --build build17max -j 2 diff --git a/docs/classes.rst b/docs/classes.rst index c0c53135..52cd52da 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -58,6 +58,16 @@ interactive Python session demonstrating this example is shown below: Static member functions can be bound in the same way using :func:`class_::def_static`. +.. note:: + + Binding C++ types in unnamed namespaces (also known as anonymous namespaces) + works reliably on many platforms, but not all. The `XFAIL_CONDITION` in + tests/test_unnamed_namespace_a.py encodes the currently known conditions. + For background see `#4319 `_. + If portability is a concern, it is therefore not recommended to bind C++ + types in unnamed namespaces. It will be safest to manually pick unique + namespace names. + Keyword and default arguments ============================= It is possible to specify keyword and default arguments using the syntax diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 8364ebd1..aaa7f868 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -123,7 +123,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1d0eb27c..11ac2215 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,6 +155,8 @@ set(PYBIND11_TEST_FILES test_tagbased_polymorphic test_thread test_union + test_unnamed_namespace_a + test_unnamed_namespace_b test_vector_unique_ptr_member test_virtual_functions) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 1b4c89ad..a92ab59a 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -319,7 +319,7 @@ def test_error_already_set_what_with_happy_exceptions( @pytest.mark.skipif( # Intentionally very specific: - "sys.version_info == (3, 12, 0, 'alpha', 6)", + "sys.version_info == (3, 12, 0, 'alpha', 7)", reason="WIP: https://github.com/python/cpython/issues/102594", ) @pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") diff --git a/tests/test_unnamed_namespace_a.cpp b/tests/test_unnamed_namespace_a.cpp new file mode 100644 index 00000000..2152e64b --- /dev/null +++ b/tests/test_unnamed_namespace_a.cpp @@ -0,0 +1,38 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_a, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_a_any_struct"); + } else { + m.attr("unnamed_namespace_a_any_struct") = py::none(); + } + m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION; + m.attr("defined_WIN32_or__WIN32") = +#if defined(WIN32) || defined(_WIN32) + true; +#else + false; +#endif + m.attr("defined___clang__") = +#if defined(__clang__) + true; +#else + false; +#endif + m.attr("defined__LIBCPP_VERSION") = +#if defined(_LIBCPP_VERSION) + true; +#else + false; +#endif + m.attr("defined___GLIBCXX__") = +#if defined(__GLIBCXX__) + true; +#else + false; +#endif +} diff --git a/tests/test_unnamed_namespace_a.py b/tests/test_unnamed_namespace_a.py new file mode 100644 index 00000000..9d9856c5 --- /dev/null +++ b/tests/test_unnamed_namespace_a.py @@ -0,0 +1,34 @@ +import pytest + +from pybind11_tests import unnamed_namespace_a as m +from pybind11_tests import unnamed_namespace_b as mb + +XFAIL_CONDITION = ( + "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))" + " or " + "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32" + " and " + "(m.defined___clang__ or m.defined__LIBCPP_VERSION))" +) +XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=False) +@pytest.mark.parametrize( + "any_struct", [m.unnamed_namespace_a_any_struct, mb.unnamed_namespace_b_any_struct] +) +def test_have_class_any_struct(any_struct): + assert any_struct is not None + + +def test_have_at_least_one_class_any_struct(): + assert ( + m.unnamed_namespace_a_any_struct is not None + or mb.unnamed_namespace_b_any_struct is not None + ) + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=True) +def test_have_both_class_any_struct(): + assert m.unnamed_namespace_a_any_struct is not None + assert mb.unnamed_namespace_b_any_struct is not None diff --git a/tests/test_unnamed_namespace_b.cpp b/tests/test_unnamed_namespace_b.cpp new file mode 100644 index 00000000..f97757a7 --- /dev/null +++ b/tests/test_unnamed_namespace_b.cpp @@ -0,0 +1,13 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_b, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_b_any_struct"); + } else { + m.attr("unnamed_namespace_b_any_struct") = py::none(); + } +} diff --git a/tests/test_unnamed_namespace_b.py b/tests/test_unnamed_namespace_b.py new file mode 100644 index 00000000..4bcaa7a6 --- /dev/null +++ b/tests/test_unnamed_namespace_b.py @@ -0,0 +1,5 @@ +from pybind11_tests import unnamed_namespace_b as m + + +def test_have_attr_any_struct(): + assert hasattr(m, "unnamed_namespace_b_any_struct") From 3f366ff888c846a3bc39af36615f99dc9e424e51 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 27 Apr 2023 07:24:48 -0700 Subject: [PATCH 02/78] Remove stray comment. (Oversight in PR #4631. Noticed by chance.) (#4641) --- tests/test_vector_unique_ptr_member.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_vector_unique_ptr_member.cpp b/tests/test_vector_unique_ptr_member.cpp index 657743e4..680cf9a6 100644 --- a/tests/test_vector_unique_ptr_member.cpp +++ b/tests/test_vector_unique_ptr_member.cpp @@ -45,8 +45,6 @@ using namespace pybind11_tests::vector_unique_ptr_member; py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } -// PYBIND11_SMART_HOLDER_TYPE_CASTERS(VectorOwner) - TEST_SUBMODULE(vector_unique_ptr_member, m) { py::class_(m, "VectorOwner") .def_static("Create", &VectorOwner::Create) From 5e946c2fa5b3c3170d702f57311492a8f838444e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Apr 2023 14:32:32 -0400 Subject: [PATCH 03/78] chore(deps): update pre-commit hooks (#4605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/psf/black: 23.1.0 → 23.3.0](https://github.com/psf/black/compare/23.1.0...23.3.0) - [github.com/Lucas-C/pre-commit-hooks: v1.4.2 → v1.5.1](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.4.2...v1.5.1) - [github.com/charliermarsh/ruff-pre-commit: v0.0.254 → v0.0.260](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.254...v0.0.260) - [github.com/PyCQA/pylint: v2.16.4 → v3.0.0a6](https://github.com/PyCQA/pylint/compare/v2.16.4...v3.0.0a6) - [github.com/pre-commit/mirrors-mypy: v1.0.1 → v1.1.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.0.1...v1.1.1) - [github.com/codespell-project/codespell: v2.2.2 → v2.2.4](https://github.com/codespell-project/codespell/compare/v2.2.2...v2.2.4) - [github.com/pre-commit/mirrors-clang-format: v15.0.7 → v16.0.0](https://github.com/pre-commit/mirrors-clang-format/compare/v15.0.7...v16.0.0) * style: pre-commit fixes * style: fix issues * Update tests/test_call_policies.py * Update tests/test_call_policies.py * fix: ignore code in file * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 18 +++---- include/pybind11/detail/type_caster_base.h | 6 +-- pyproject.toml | 1 + tests/test_iostream.py | 58 +++++++++++----------- tests/test_stl.py | 2 +- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c431619..51b6f0a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "23.1.0" # Keep in sync with blacken-docs + rev: "23.3.0" # Keep in sync with blacken-docs hooks: - id: black @@ -51,11 +51,11 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.1.0 # keep in sync with black hook + - black==23.3.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.4.2" + rev: "v1.5.1" hooks: - id: remove-tabs @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.254 + rev: v0.0.263 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -84,7 +84,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v2.16.4" + rev: "v3.0.0a6" hooks: - id: pylint files: ^pybind11 @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.0.1" + rev: "v1.1.1" hooks: - id: mypy args: [] @@ -120,11 +120,11 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.2" + rev: "v2.2.4" hooks: - id: codespell exclude: ".supp$" - args: ["-x", ".codespell-ignore-lines"] + args: ["-x.codespell-ignore-lines", "-Lccompiler"] # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v15.0.7" + rev: "v16.0.0" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 34dcd26c..8f3d0f37 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -258,9 +258,9 @@ struct value_and_holder { // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; diff --git a/pyproject.toml b/pyproject.toml index 96cf8426..e3655aca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,3 +93,4 @@ isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests [tool.ruff.per-file-ignores] "tests/**" = ["EM", "N"] +"tests/test_call_policies.py" = ["PLC1901"] diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 79b2a2b8..d283eb15 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -9,16 +9,16 @@ def test_captured(capsys): m.captured_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_err(msg) stdout, stderr = capsys.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg @@ -30,7 +30,7 @@ def test_captured_large_string(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset0(capsys): @@ -40,7 +40,7 @@ def test_captured_utf8_2byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset1(capsys): @@ -50,7 +50,7 @@ def test_captured_utf8_2byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset0(capsys): @@ -60,7 +60,7 @@ def test_captured_utf8_3byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset1(capsys): @@ -70,7 +70,7 @@ def test_captured_utf8_3byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset2(capsys): @@ -80,7 +80,7 @@ def test_captured_utf8_3byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset0(capsys): @@ -90,7 +90,7 @@ def test_captured_utf8_4byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset1(capsys): @@ -100,7 +100,7 @@ def test_captured_utf8_4byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset2(capsys): @@ -110,7 +110,7 @@ def test_captured_utf8_4byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset3(capsys): @@ -120,7 +120,7 @@ def test_captured_utf8_4byte_offset3(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_guard_capture(capsys): @@ -128,7 +128,7 @@ def test_guard_capture(capsys): m.guard_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_series_captured(capture): @@ -145,7 +145,7 @@ def test_flush(capfd): with m.ostream_redirect(): m.noisy_function(msg, flush=False) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout m.noisy_function(msg2, flush=True) stdout, stderr = capfd.readouterr() @@ -164,15 +164,15 @@ def test_not_captured(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" - assert stream.getvalue() == "" + assert not stderr + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream): m.captured_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -182,16 +182,16 @@ def test_err(capfd): with redirect_stderr(stream): m.raw_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stderr(stream): m.captured_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -221,13 +221,13 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream), m.ostream_redirect(): m.raw_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stream.getvalue() == msg stream = StringIO() @@ -235,7 +235,7 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() def test_redirect_err(capfd): @@ -248,7 +248,7 @@ def test_redirect_err(capfd): m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr assert stream.getvalue() == msg2 @@ -262,8 +262,8 @@ def test_redirect_both(capfd): m.raw_output(msg) m.raw_err(msg2) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg assert stream2.getvalue() == msg2 diff --git a/tests/test_stl.py b/tests/test_stl.py index b7a0d1c0..2d9dcc89 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -367,7 +367,7 @@ def test_issue_1561(): """check fix for issue #1561""" bar = m.Issue1561Outer() bar.list = [m.Issue1561Inner("bar")] - bar.list + assert bar.list assert bar.list[0].data == "bar" From 956390a87f6e2ee6ea20438f3faa05222415018f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 29 Apr 2023 10:15:45 -0400 Subject: [PATCH 04/78] fix(cmake): only define lto if CMAKE's IPO setting is unset (#4643) Signed-off-by: Henry Schreiner --- tools/pybind11Common.cmake | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 0c985bc8..e4ff1e40 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -5,8 +5,8 @@ Adds the following targets:: pybind11::pybind11 - link to headers and pybind11 pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -20,7 +20,7 @@ Adds the following functions:: # CMake 3.10 has an include_guard command, but we can't use that yet # include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::lto) +if(TARGET pybind11::pybind11) return() endif() @@ -372,11 +372,13 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- From da91926295638b2b1c0c85a568a2d66ca3646565 Mon Sep 17 00:00:00 2001 From: biergaizi Date: Mon, 1 May 2023 14:14:52 +0000 Subject: [PATCH 05/78] fix: remove -stdlib=libc++ from setup helpers, not needed on modern Pythons (#4639) * Inject -stdlib=libc++ on macOS only when it's supported, close #4637. On macOS, by default, pybind11 currently unconditionally set the compiler flag "-stdlib=libc++" in Pybind11Extension.__init__(), regardless of which compiler is used. This flag is required for clang, but is invalid for GCC. If GCC is used, it causes compilation failures in all Python projects that use pybind11, with the error message: arm64-apple-darwin22-gcc: error: unrecognized command-line option -stdlib=libc++. This commit uses has_flag() to detect whether "-stdlib=libc++" on macOS, and injects this flag from build_ext.build_extensions(), rather than setting it unconditionally. Signed-off-by: Yifeng Li * revert: just remove flags --------- Signed-off-by: Yifeng Li Co-authored-by: Henry Schreiner --- pybind11/setup_helpers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 0fb4679e..cb279f27 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -144,7 +144,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: @@ -154,11 +153,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property def cxx_std(self) -> int: From dff75a62bd33ff35312aedf696c17f0300301b89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:34:51 -0400 Subject: [PATCH 06/78] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.5 to 1.8.6 (#4650) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.5 to 1.8.6. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.5...v1.8.6) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 7f686b13..6d9be3b1 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -98,13 +98,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.5 + uses: pypa/gh-action-pypi-publish@v1.8.6 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.5 + uses: pypa/gh-action-pypi-publish@v1.8.6 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From b3e88ecf894269b6b3fe822a7551c95247a3c47e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 00:35:21 -0400 Subject: [PATCH 07/78] chore(deps): update pre-commit hooks (#4648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.1.1 → v1.2.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.1.1...v1.2.0) - [github.com/pre-commit/mirrors-clang-format: v16.0.0 → v16.0.2](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.0...v16.0.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51b6f0a4..aca9b3ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.1.1" + rev: "v1.2.0" hooks: - id: mypy args: [] @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.0" + rev: "v16.0.2" hooks: - id: clang-format types_or: [c++, c, cuda] From f70165463328c218d118204efc13aac93783d17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 5 May 2023 07:39:05 +0200 Subject: [PATCH 08/78] Introduce recursive_container_traits (#4623) * Testing * Similar fix for std::vector * Fix infinite recursion check: 1) Apply to is_copy_assignable additionally 2) Check infinite recursion for map-like types * style: pre-commit fixes * Optional commit that demonstrates the limitations of this PR * Fix positioning of container bindings The bindings were previously in a block that was only activated if numpy was available. * Suggestions from code review: API side * Suggestions from code review: Test side * Suggestions from code review 1) Renaming: is_recursive_container and MutuallyRecursiveContainerPair(MV|VM) 2) Avoid ambiguous specializations of is_recursive_container * Some little fixes * Reordering of structs * Add recursive checks for is_move_constructible * Static testing for pybind11 type traits * More precise checking of recursive types Instead of a trait `is_recursive_container`, use a trait `recursive_container_traits` with dependent type `recursive_container_traits::type_to_check_recursively`. So, instead of just checking if a type is recursive and then trying to somehow deal with it, recursively-defined traits such as is_move_constructible can now directly ask this trait where the recursion should proceed. * Review suggestions 1. Use std::conditional 2. Fix typo * Remove leftover include from test --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/type_caster_base.h | 203 +++++++++++++++--- include/pybind11/stl_bind.h | 8 +- tests/test_copy_move.cpp | 238 +++++++++++++++++++++ tests/test_stl_binders.cpp | 44 ++++ tests/test_stl_binders.py | 18 ++ 5 files changed, 484 insertions(+), 27 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 8f3d0f37..16387506 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -822,26 +822,179 @@ using movable_cast_op_type typename std::add_rvalue_reference>::type, typename std::add_lvalue_reference>::type>>; -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template -struct is_copy_constructible : std::is_copy_constructible {}; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template +struct impl_type_to_check_recursively, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const::type; + using if_not_recursive = std::pair::type, B>; +}; -template -struct is_move_constructible : std::is_move_constructible {}; +/* + * Implementation of `recursive_container_traits` below. + */ +template +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. template -struct is_copy_constructible< +struct impl_recursive_container_traits< Container, - enable_if_t< - all_of, - std::is_same, - // Avoid infinite recursion - negation>>::value>> - : is_copy_constructible {}; + typename std::enable_if::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits::has_recursive_mapped_type + || container_value_type_traits::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template +struct recursive_container_traits : impl_recursive_container_traits {}; + +template +struct is_move_constructible + : all_of, + is_move_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template +struct is_move_constructible> + : all_of, is_move_constructible> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template +struct is_copy_constructible + : all_of, + is_copy_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -852,14 +1005,16 @@ struct is_copy_constructible> : all_of, is_copy_constructible> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template -struct is_copy_assignable : std::is_copy_assignable {}; -template -struct is_copy_assignable, - std::is_same>::value>> - : is_copy_assignable {}; +template +struct is_copy_assignable + : all_of< + std::is_copy_assignable, + is_copy_assignable::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable : std::true_type {}; + template struct is_copy_assignable> : all_of, is_copy_assignable> {}; diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 7cfd996d..49f1b778 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -61,9 +61,11 @@ struct is_comparable< /* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ template -struct is_comparable::is_vector>> { - static constexpr const bool value = is_comparable::value; -}; +struct is_comparable::is_vector>> + : is_comparable::type_to_check_recursively> {}; + +template <> +struct is_comparable : std::true_type {}; /* For pairs, recursively check the two data types */ template diff --git a/tests/test_copy_move.cpp b/tests/test_copy_move.cpp index 28c24456..f5473355 100644 --- a/tests/test_copy_move.cpp +++ b/tests/test_copy_move.cpp @@ -13,6 +13,8 @@ #include "constructor_stats.h" #include "pybind11_tests.h" +#include + template struct empty { static const derived &get_one() { return instance_; } @@ -293,3 +295,239 @@ TEST_SUBMODULE(copy_move_policies, m) { // Make sure that cast from pytype rvalue to other pytype works m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast(); }); } + +/* + * Rest of the file: + * static_assert based tests for pybind11 adaptations of + * std::is_move_constructible, std::is_copy_constructible and + * std::is_copy_assignable (no adaptation of std::is_move_assignable). + * Difference between pybind11 and std traits: pybind11 traits will also check + * the contained value_types. + */ + +struct NotMovable { + NotMovable() = default; + NotMovable(NotMovable const &) = default; + NotMovable(NotMovable &&) = delete; + NotMovable &operator=(NotMovable const &) = default; + NotMovable &operator=(NotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyable { + NotCopyable() = default; + NotCopyable(NotCopyable const &) = delete; + NotCopyable(NotCopyable &&) = default; + NotCopyable &operator=(NotCopyable const &) = delete; + NotCopyable &operator=(NotCopyable &&) = default; +}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovable { + NotCopyableNotMovable() = default; + NotCopyableNotMovable(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable(NotCopyableNotMovable &&) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct RecursiveVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct RecursiveMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index ca9630bd..1681760a 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -70,6 +70,44 @@ NestMap *times_hundred(int n) { return m; } +/* + * Recursive data structures as test for issue #4623 + */ +struct RecursiveVector : std::vector { + using Parent = std::vector; + using Parent::Parent; +}; + +struct RecursiveMap : std::map { + using Parent = std::map; + using Parent::Parent; +}; + +/* + * Pybind11 does not catch more complicated recursion schemes, such as mutual + * recursion. + * In that case custom recursive_container_traits specializations need to be added, + * thus manually telling pybind11 about the recursion. + */ +struct MutuallyRecursiveContainerPairMV; +struct MutuallyRecursiveContainerPairVM; + +struct MutuallyRecursiveContainerPairMV : std::map {}; +struct MutuallyRecursiveContainerPairVM : std::vector {}; + +namespace pybind11 { +namespace detail { +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(stl_binders, m) { // test_vector_int py::bind_vector>(m, "VectorInt", py::buffer_protocol()); @@ -129,6 +167,12 @@ TEST_SUBMODULE(stl_binders, m) { m, "VectorUndeclStruct", py::buffer_protocol()); }); + // Bind recursive container types + py::bind_vector(m, "RecursiveVector"); + py::bind_map(m, "RecursiveMap"); + py::bind_map(m, "MutuallyRecursiveContainerPairMV"); + py::bind_vector(m, "MutuallyRecursiveContainerPairVM"); + // The rest depends on numpy: try { py::module_::import("numpy"); diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 7dca742a..e002f5b6 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -335,3 +335,21 @@ def test_map_view_types(): assert type(unordered_map_string_double.items()) is items_type assert type(map_string_double_const.items()) is items_type assert type(unordered_map_string_double_const.items()) is items_type + + +def test_recursive_vector(): + recursive_vector = m.RecursiveVector() + recursive_vector.append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + # Can't use len() since test_stl_binders.cpp does not include stl.h, + # so the necessary conversion is missing + assert recursive_vector[0].count(m.RecursiveVector()) == 2 + + +def test_recursive_map(): + recursive_map = m.RecursiveMap() + recursive_map[100] = m.RecursiveMap() + recursive_map[100][101] = m.RecursiveMap() + recursive_map[100][102] = m.RecursiveMap() + assert list(recursive_map[100].keys()) == [101, 102] From 90312a6ee8de69f582e165b25338ead4f1a6ccc2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 7 May 2023 10:15:53 -0700 Subject: [PATCH 09/78] Add `type_caster` (#4601) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add `type_caster` (tests are still incomplete). * Fix oversight (`const PyObject *`). * Ensure `type_caster` only works for `PyObject *` * Move `is_same_ignoring_cvref` into `detail` namespace. * Add test_cast_nullptr * Change is_same_ignoring_cvref from variable template to using. ``` test_type_caster_pyobject_ptr.cpp:8:23: error: variable templates only available with ‘-std=c++14’ or ‘-std=gnu++14’ [-Werror] 8 | static constexpr bool is_same_ignoring_cvref = std::is_same, U>::value; | ^~~~~~~~~~~~~~~~~~~~~~ ``` * Remove `return_value_policy::reference_internal` `keep_alive` feature (because of doubts about it actually being useful). * Add missing test, fix bug (missing `throw error_already_set();`), various cosmetic changes. * Move `type_caster` from test to new include (pybind11/type_caster_pyobject_ptr.h) * Add new header file to CMakeLists.txt and tests/extra_python_package/test_files.py * Backport changes from https://github.com/google/pywrapcc/pull/30021 to https://github.com/pybind/pybind11/pull/4601 * Fix oversight in test (to resolve a valgrind leak detection error) and add a related comment in cast.h. No production code changes. Make tests more sensitive by using `ValueHolder` instead of empty tuples and dicts. Manual leak checks with `while True:` & top command repeated for all tests. * Add tests for interop with stl.h `list_caster` (No production code changes.) * Bug fix in test. Minor comment enhancements. * Change `type_caster::name` to `object`, as suggested by @Skylion007 * Expand comment for the new `T cast(const handle &handle)` [`T` = `PyObject *`] * Add `T cast(object &&obj)` overload as suggested by @Skylion007 The original suggestion leads to `error: call to 'cast' is ambiguous` (full error message below), therefore SFINAE guarding is needed. ``` clang++ -o pybind11/tests/test_type_caster_pyobject_ptr.os -c -std=c++17 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.10 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:1: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:12: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:13: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/detail/class.h:12: In file included from /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/attr.h:14: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1165:12: error: call to 'cast' is ambiguous return pybind11::cast(std::move(*this)); ^~~~~~~~~~~~~~~~~ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:109:70: note: in instantiation of function template specialization 'pybind11::object::cast<_object *>' requested here return hfunc.f(std::forward(args)...).template cast(); ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/functional.h:103:16: note: in instantiation of member function 'pybind11::detail::type_caster>::load(pybind11::handle, bool)::func_wrapper::operator()' requested here struct func_wrapper { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1456:47: note: in instantiation of member function 'pybind11::detail::type_caster>::load' requested here if ((... || !std::get(argcasters).load(call.args[Is], call.args_convert[Is]))) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1434:50: note: in instantiation of function template specialization 'pybind11::detail::argument_loader &, int>::load_impl_sequence<0UL, 1UL>' requested here bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:227:33: note: in instantiation of member function 'pybind11::detail::argument_loader &, int>::load_args' requested here if (!args_converter.load_args(call)) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:101:9: note: in instantiation of function template specialization 'pybind11::cpp_function::initialize<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), _object *, const std::function<_object *(int)> &, int, pybind11::name, pybind11::scope, pybind11::sibling, pybind11::return_value_policy>' requested here initialize( ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/pybind11.h:1163:22: note: in instantiation of function template specialization 'pybind11::cpp_function::cpp_function<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), pybind11::name, pybind11::scope, pybind11::sibling, pybind11::return_value_policy, void>' requested here cpp_function func(std::forward(f), ^ /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:48:7: note: in instantiation of function template specialization 'pybind11::module_::def<(lambda at /usr/local/google/home/rwgk/forked/pybind11/tests/test_type_caster_pyobject_ptr.cpp:50:9), pybind11::return_value_policy>' requested here m.def( ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1077:3: note: candidate function [with T = _object *, $1 = 0] T cast(object &&obj) { ^ /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/cast.h:1149:1: note: candidate function [with T = _object *] cast(object &&object) { ^ 1 error generated. ``` --- CMakeLists.txt | 3 +- include/pybind11/cast.h | 34 ++++- include/pybind11/detail/common.h | 4 + include/pybind11/type_caster_pyobject_ptr.h | 61 +++++++++ tests/CMakeLists.txt | 1 + tests/extra_python_package/test_files.py | 1 + tests/test_type_caster_pyobject_ptr.cpp | 130 ++++++++++++++++++++ tests/test_type_caster_pyobject_ptr.py | 104 ++++++++++++++++ 8 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 include/pybind11/type_caster_pyobject_ptr.h create mode 100644 tests/test_type_caster_pyobject_ptr.cpp create mode 100644 tests/test_type_caster_pyobject_ptr.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d932038..25a7d149 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,8 @@ set(PYBIND11_HEADERS include/pybind11/pytypes.h include/pybind11/stl.h include/pybind11/stl_bind.h - include/pybind11/stl/filesystem.h) + include/pybind11/stl/filesystem.h + include/pybind11/type_caster_pyobject_ptr.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9d1ce15d..db393411 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1041,7 +1041,11 @@ make_caster load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template ::value, int> = 0> +template ::value + && !detail::is_same_ignoring_cvref::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference::value, @@ -1055,6 +1059,34 @@ T cast(const handle &handle) { return T(reinterpret_borrow(handle)); } +// Note that `cast(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template ::value, int> = 0> object cast(T &&value, diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 12903925..1ff0df09 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -661,6 +661,10 @@ template using remove_cvref_t = typename remove_cvref::type; #endif +/// Example usage: is_same_ignoring_cvref::value +template +using is_same_ignoring_cvref = std::is_same, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 00000000..aa914f9e --- /dev/null +++ b/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template ::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref::value, + "Invalid C++ type T for to-Python conversion (type_caster)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster::cast(): unsupported return_value_policy: " + + std::to_string(static_cast(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow(src); + return true; + } + + template + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 11ac2215..5574663b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -154,6 +154,7 @@ set(PYBIND11_TEST_FILES test_stl_binders test_tagbased_polymorphic test_thread + test_type_caster_pyobject_ptr test_union test_unnamed_namespace_a test_unnamed_namespace_b diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index dd6393bf..e5982f96 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -43,6 +43,7 @@ "include/pybind11/pytypes.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/type_caster_pyobject_ptr.h", } detail_headers = { diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp new file mode 100644 index 00000000..1667ea12 --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.cpp @@ -0,0 +1,130 @@ +#include +#include +#include + +#include "pybind11_tests.h" + +#include +#include + +namespace { + +std::vector make_vector_pyobject_ptr(const py::object &ValueHolder) { + std::vector vec_obj; + for (int i = 1; i < 3; i++) { + vec_obj.push_back(ValueHolder(i * 93).release().ptr()); + } + // This vector now owns the refcounts. + return vec_obj; +} + +} // namespace + +TEST_SUBMODULE(type_caster_pyobject_ptr, m) { + m.def("cast_from_pyobject_ptr", []() { + PyObject *ptr = PyLong_FromLongLong(6758L); + return py::cast(ptr, py::return_value_policy::take_ownership); + }); + m.def("cast_handle_to_pyobject_ptr", [](py::handle obj) { + auto rc1 = obj.ref_count(); + auto *ptr = py::cast(obj); + auto rc2 = obj.ref_count(); + if (rc2 != rc1 + 1) { + return -1; + } + return 100 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_object_to_pyobject_ptr", [](py::object obj) { + py::handle hdl = obj; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(obj)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 300 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_list_to_pyobject_ptr", [](py::list lst) { + // This is to cover types implicitly convertible to object. + py::handle hdl = lst; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(lst)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 400 - static_cast(py::len(py::reinterpret_steal(ptr))); + }); + + m.def( + "return_pyobject_ptr", + []() { return PyLong_FromLongLong(2314L); }, + py::return_value_policy::take_ownership); + m.def("pass_pyobject_ptr", [](PyObject *ptr) { + return 200 - py::reinterpret_borrow(ptr).attr("value").cast(); + }); + + m.def("call_callback_with_object_return", + [](const std::function &cb, int value) { return cb(value); }); + m.def( + "call_callback_with_pyobject_ptr_return", + [](const std::function &cb, int value) { return cb(value); }, + py::return_value_policy::take_ownership); + m.def( + "call_callback_with_pyobject_ptr_arg", + [](const std::function &cb, py::handle obj) { return cb(obj.ptr()); }, + py::arg("cb"), // This triggers return_value_policy::automatic_reference + py::arg("obj")); + + m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) { + if (set_error) { + PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling."); + } + PyObject *ptr = nullptr; + py::cast(ptr); + }); + + m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() { + PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling."); + py::cast(Py_None); + }); + + m.def("pass_list_pyobject_ptr", [](const std::vector &vec_obj) { + int acc = 0; + for (const auto &ptr : vec_obj) { + acc = acc * 1000 + py::reinterpret_borrow(ptr).attr("value").cast(); + } + return acc; + }); + + m.def("return_list_pyobject_ptr_take_ownership", + make_vector_pyobject_ptr, + // Ownership is transferred one-by-one when the vector is converted to a Python list. + py::return_value_policy::take_ownership); + + m.def("return_list_pyobject_ptr_reference", + make_vector_pyobject_ptr, + // Ownership is not transferred. + py::return_value_policy::reference); + + m.def("dec_ref_each_pyobject_ptr", [](const std::vector &vec_obj) { + std::size_t i = 0; + for (; i < vec_obj.size(); i++) { + py::handle h(vec_obj[i]); + if (static_cast(h.ref_count()) < 2) { + break; // Something is badly wrong. + } + h.dec_ref(); + } + return i; + }); + + m.def("pass_pyobject_ptr_and_int", [](PyObject *, int) {}); + +#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing. + { + PyObject *ptr = nullptr; + (void) py::cast(*ptr); + } +#endif +} diff --git a/tests/test_type_caster_pyobject_ptr.py b/tests/test_type_caster_pyobject_ptr.py new file mode 100644 index 00000000..1f1ece2b --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.py @@ -0,0 +1,104 @@ +import pytest + +from pybind11_tests import type_caster_pyobject_ptr as m + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below. +class ValueHolder: + def __init__(self, value): + self.value = value + + +def test_cast_from_pyobject_ptr(): + assert m.cast_from_pyobject_ptr() == 6758 + + +def test_cast_handle_to_pyobject_ptr(): + assert m.cast_handle_to_pyobject_ptr(ValueHolder(24)) == 76 + + +def test_cast_object_to_pyobject_ptr(): + assert m.cast_object_to_pyobject_ptr(ValueHolder(43)) == 257 + + +def test_cast_list_to_pyobject_ptr(): + assert m.cast_list_to_pyobject_ptr([1, 2, 3, 4, 5]) == 395 + + +def test_return_pyobject_ptr(): + assert m.return_pyobject_ptr() == 2314 + + +def test_pass_pyobject_ptr(): + assert m.pass_pyobject_ptr(ValueHolder(82)) == 118 + + +@pytest.mark.parametrize( + "call_callback", + [ + m.call_callback_with_object_return, + m.call_callback_with_pyobject_ptr_return, + ], +) +def test_call_callback_with_object_return(call_callback): + def cb(value): + if value < 0: + raise ValueError("Raised from cb") + return ValueHolder(1000 - value) + + assert call_callback(cb, 287).value == 713 + + with pytest.raises(ValueError, match="^Raised from cb$"): + call_callback(cb, -1) + + +def test_call_callback_with_pyobject_ptr_arg(): + def cb(obj): + return 300 - obj.value + + assert m.call_callback_with_pyobject_ptr_arg(cb, ValueHolder(39)) == 261 + + +@pytest.mark.parametrize("set_error", [True, False]) +def test_cast_to_python_nullptr(set_error): + expected = { + True: r"^Reflective of healthy error handling\.$", + False: ( + r"^Internal error: pybind11::error_already_set called " + r"while Python error indicator not set\.$" + ), + }[set_error] + with pytest.raises(RuntimeError, match=expected): + m.cast_to_pyobject_ptr_nullptr(set_error) + + +def test_cast_to_python_non_nullptr_with_error_set(): + with pytest.raises(SystemError) as excinfo: + m.cast_to_pyobject_ptr_non_nullptr_with_error_set() + assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()" + assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling." + + +def test_pass_list_pyobject_ptr(): + acc = m.pass_list_pyobject_ptr([ValueHolder(842), ValueHolder(452)]) + assert acc == 842452 + + +def test_return_list_pyobject_ptr_take_ownership(): + vec_obj = m.return_list_pyobject_ptr_take_ownership(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + + +def test_return_list_pyobject_ptr_reference(): + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + +def test_type_caster_name_via_incompatible_function_arguments_type_error(): + with pytest.raises(TypeError, match=r"1\. \(arg0: object, arg1: int\) -> None"): + m.pass_pyobject_ptr_and_int(ValueHolder(101), ValueHolder(202)) From e9b961d9b913575c07ba28c038c3706731768da6 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 8 May 2023 10:13:54 -0700 Subject: [PATCH 10/78] Elide to-python conversion of setter return values (#4621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reproducer for property setter with return type that is not wrapped. * Use `py::class_()` to work around the return value conversion issue. * WIP drop_return_value * Remove struct drop_return_value * Introduce `return_value_policy::return_none` for use by setters. * Add `is_setter` to attr.h and use from `.def_property()` * Merge the new test into test_methods_and_attributes * Remove return_none return_value_policy again. * Fix oversight (NOLINTNEXTLINE placement). * Simplification (for the better) found while searching for a way to resolve GCC build failures. Example of failure resolved by this change: g++ (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0 ``` cd /build/tests && /usr/bin/c++ -DPYBIND11_TEST_EIGEN -Dpybind11_tests_EXPORTS -I/mounted_pybind11/include -isystem /usr/include/python3.8 -isystem /build/_deps/eigen-src -g -std=c++17 -fPIC -fvisibility=hidden -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -MD -MT tests/CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o -MF CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o.d -o CMakeFiles/pybind11_tests.dir/test_buffers.cpp.o -c /mounted_pybind11/tests/test_buffers.cpp In file included from /mounted_pybind11/include/pybind11/stl.h:12, from /mounted_pybind11/tests/test_buffers.cpp:10: /mounted_pybind11/include/pybind11/pybind11.h: In instantiation of ‘pybind11::class_& pybind11::class_::def_property(const char*, const Getter&, const Setter&, const Extra& ...) [with Getter = pybind11::cpp_function; Setter = std::nullptr_t; Extra = {pybind11::return_value_policy}; type_ = pybind11::buffer_info; options = {}]’: /mounted_pybind11/include/pybind11/pybind11.h:1716:58: required from ‘pybind11::class_& pybind11::class_::def_property_readonly(const char*, const pybind11::cpp_function&, const Extra& ...) [with Extra = {pybind11::return_value_policy}; type_ = pybind11::buffer_info; options = {}]’ /mounted_pybind11/include/pybind11/pybind11.h:1684:9: required from ‘pybind11::class_& pybind11::class_::def_readonly(const char*, const D C::*, const Extra& ...) [with C = pybind11::buffer_info; D = long int; Extra = {}; type_ = pybind11::buffer_info; options = {}]’ /mounted_pybind11/tests/test_buffers.cpp:209:61: required from here /mounted_pybind11/include/pybind11/pybind11.h:1740:25: error: call of overloaded ‘cpp_function(std::nullptr_t&, pybind11::is_setter)’ is ambiguous 1740 | name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /mounted_pybind11/include/pybind11/pybind11.h:101:5: note: candidate: ‘pybind11::cpp_function::cpp_function(Func&&, const Extra& ...) [with Func = std::nullptr_t&; Extra = {pybind11::is_setter}; = void]’ 101 | cpp_function(Func &&f, const Extra &...extra) { | ^~~~~~~~~~~~ In file included from /mounted_pybind11/include/pybind11/stl.h:12, from /mounted_pybind11/tests/test_buffers.cpp:10: /mounted_pybind11/include/pybind11/pybind11.h:87:5: note: candidate: ‘pybind11::cpp_function::cpp_function(std::nullptr_t, const Extra& ...) [with Extra = {pybind11::is_setter}; std::nullptr_t = std::nullptr_t]’ 87 | cpp_function(std::nullptr_t, const Extra &...) {} | ^~~~~~~~~~~~ ``` * Bug fix: obvious in hindsight. I thought the original version was incrementing the reference count for None, but no. Discovered via many failing tests in the wild (10s of thousands). It is very tricky to construct a meaningful unit test for this bug specifically. It's unlikely to come back, because 10s of thousands of tests will fail again. --- include/pybind11/attr.h | 16 +++++++++++-- include/pybind11/pybind11.h | 18 ++++++++++---- tests/test_methods_and_attributes.cpp | 34 +++++++++++++++++++++++++++ tests/test_methods_and_attributes.py | 9 +++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index b5e3b7b2..1044db94 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -26,6 +26,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -188,8 +191,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +233,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -426,6 +432,12 @@ struct process_attribute : process_attribute_default { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute : process_attribute_default { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 6205effd..28ebc222 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -84,6 +84,7 @@ class cpp_function : public function { cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template @@ -244,10 +245,16 @@ class cpp_function : public function { using Guard = extract_guard_t; /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call(cap->f), - policy, - call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes::postcall(call, result); @@ -1729,7 +1736,8 @@ class class_ : public detail::generic_type { template class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); } template class_ &def_property(const char *name, diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 815dd5e9..31d46eb7 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -177,6 +177,38 @@ struct RValueRefParam { std::size_t func4(std::string &&s) const & { return s.size(); } }; +namespace pybind11_tests { +namespace exercise_is_setter { + +struct FieldBase { + int int_value() const { return int_value_; } + + FieldBase &SetIntValue(int int_value) { + int_value_ = int_value; + return *this; + } + +private: + int int_value_ = -99; +}; + +struct Field : FieldBase {}; + +void add_bindings(py::module &m) { + py::module sm = m.def_submodule("exercise_is_setter"); + // NOTE: FieldBase is not wrapped, therefore ... + py::class_(sm, "Field") + .def(py::init<>()) + .def_property( + "int_value", + &Field::int_value, + &Field::SetIntValue // ... the `FieldBase &` return value here cannot be converted. + ); +} + +} // namespace exercise_is_setter +} // namespace pybind11_tests + TEST_SUBMODULE(methods_and_attributes, m) { // test_methods_and_attributes py::class_ emna(m, "ExampleMandA"); @@ -456,4 +488,6 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def("func2", &RValueRefParam::func2) .def("func3", &RValueRefParam::func3) .def("func4", &RValueRefParam::func4); + + pybind11_tests::exercise_is_setter::add_bindings(m); } diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index a8546857..955a85f6 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -522,3 +522,12 @@ def test_rvalue_ref_param(): assert r.func2("1234") == 4 assert r.func3("12345") == 5 assert r.func4("123456") == 6 + + +def test_is_setter(): + fld = m.exercise_is_setter.Field() + assert fld.int_value == -99 + setter_return = fld.int_value = 100 + assert isinstance(setter_return, int) + assert setter_return == 100 + assert fld.int_value == 100 From cca4c51ca463ea02fa504331ff21bc313c80c7f3 Mon Sep 17 00:00:00 2001 From: Tim Stumbaugh Date: Tue, 9 May 2023 08:04:20 -0600 Subject: [PATCH 11/78] Update errors in string "Explicit conversions" docs (#4658) `PyUnicode_DecodeLatin1` requires you to pass in the `error` parameter. The code as it is in the docs didn't compile. There is a reference leak in the example code. `PyUnicode_DecodeLatin1` returns a new reference. Calling `py::str(PyObject*)` calls `PyObject_Str`, which also returns a new reference. That reference is managed by the `py::str` constructor (which correctly steals the reference, using the `stolen_t` constructor), but the original reference returned by `PyUnicode_DecodeLatin1` is never decref'd: it never makes it into an `object`, and it's never manually decremented. This fixes both of those issues. The code compiles, and I viewed the sphinx docs locally. --- docs/advanced/cast/strings.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/advanced/cast/strings.rst b/docs/advanced/cast/strings.rst index e246c521..271716b4 100644 --- a/docs/advanced/cast/strings.rst +++ b/docs/advanced/cast/strings.rst @@ -101,8 +101,11 @@ conversion has the same overhead as implicit conversion. m.def("str_output", []() { std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1 - py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length()); - return py_s; + py::handle py_s = PyUnicode_DecodeLatin1(s.data(), s.length(), nullptr); + if (!py_s) { + throw py::error_already_set(); + } + return py::reinterpret_steal(py_s); } ); @@ -113,7 +116,8 @@ conversion has the same overhead as implicit conversion. The `Python C API `_ provides -several built-in codecs. +several built-in codecs. Note that these all return *new* references, so +use :cpp:func:`reinterpret_steal` when converting them to a :cpp:class:`str`. One could also use a third party encoding library such as libiconv to transcode From d72ffb448c58b4ffb08b5ad629bc788646e2d59e Mon Sep 17 00:00:00 2001 From: Joyce Date: Mon, 15 May 2023 14:02:25 -0300 Subject: [PATCH 12/78] ci: set minimal permissions to github workflows (#4665) * set ci.yml minimal permissions Signed-off-by: Joyce * set configure.yml minimal permissions Signed-off-by: Joyce * set format.yml minimal permissions Signed-off-by: Joyce * set pip.yml minimal permissions Signed-off-by: Joyce * set upstream.yml minimal permissions Signed-off-by: Joyce * set labeler.yml minimal permissions Signed-off-by: Joyce * Update ci.yml to read all Signed-off-by: Joyce * test labeler.yml Signed-off-by: Joyce * restore the if at labeler.yml Signed-off-by: Joyce --------- Signed-off-by: Joyce --- .github/workflows/ci.yml | 2 ++ .github/workflows/configure.yml | 3 +++ .github/workflows/format.yml | 3 +++ .github/workflows/labeler.yml | 5 +++++ .github/workflows/pip.yml | 3 +++ .github/workflows/upstream.yml | 3 +++ 6 files changed, 19 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c88f0797..8c2aba34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,8 @@ on: - stable - v* +permissions: read-all + concurrency: group: test-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index b469a69d..4ae22281 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -9,6 +9,9 @@ on: - stable - v* +permissions: + contents: read + env: # For cmake: VERBOSE: 1 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 46489feb..b8242ee5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -12,6 +12,9 @@ on: - stable - "v*" +permissions: + contents: read + env: FORCE_COLOR: 3 # For cmake: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 165a2fd8..858a4a0e 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,10 +3,15 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: label: name: Labeler runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: actions/labeler@main diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 6d9be3b1..c1feb6fe 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -12,6 +12,9 @@ on: types: - published +permissions: + contents: read + env: PIP_ONLY_BINARY: numpy diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index be643ddf..4acfbfce 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -5,6 +5,9 @@ on: workflow_dispatch: pull_request: +permissions: + contents: read + concurrency: group: upstream-${{ github.ref }} cancel-in-progress: true From 19816f0db7b89468104f5ed284c0ec9d7e312c44 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 22 May 2023 13:44:03 -0700 Subject: [PATCH 13/78] chore: update changelog, with target date for v2.11.0 release (#4677) * Remove .dev1 from version number. * [skip ci] Update changelog.rst * [ci skip] Fix pre-commit: rst ``code`` is two backticks * Apply fix suggested by @henryiii * [ci skip] Set target date for the release to June 2, 2023 * [ci skip] Apply more fixes suggested by @henryiii (I missed those before). * [ci skip] Revert "Remove .dev1 from version number." This reverts commit afc80134cb1035be2913fd8153a16a864181c52c. --- docs/changelog.rst | 54 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 58cc4098..32f6606a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,10 +10,25 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. -IN DEVELOPMENT --------------- +Version 2.11.0 (June 2, 2023) +----------------------------- + +New features: + +* ``pybind11::detail::is_move_constructible`` can now be specialized for cases + in which ``std::is_move_constructible`` does not work as needed. This is + very similar to the long-established + ``pybind11::detail::is_copy_constructible``. + `#4631 `_ -Changes will be summarized here periodically. +* Introduce ``recursive_container_traits``. + `#4623 `_ + +* ``pybind11/type_caster_pyobject_ptr.h`` was added to support automatic + wrapping of APIs that make use of ``PyObject *``. This header needs to + included explicitly (i.e. it is not included implicitly + with ``pybind/pybind11.h``). + `#4601 `_ Changes: @@ -31,6 +46,28 @@ Changes: sizes slightly (~1.5%) but the error messages are much more informative. `#4463 `_ +* Setter return values (which are inaccessible for all practical purposes) are + no longer converted to Python (only to be discarded). + `#4621 `_ + +* Allow lambda specified to function definition to be ``noexcept(true)`` + in C++17. + `#4593 `_ + +* Get rid of recursive template instantiations for concatenating type + signatures on C++17 and higher. + `#4587 `_ + +* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11 + ABI version for Python 3.12 is version 5. (The default ABI version + for Python versions up to and including 3.11 is still version 4.). + `#4570 `_ + +* With ``PYBIND11_INTERNALS_VERSION 5`` (default for Python 3.12+), MSVC builds + use ``std::hash`` and ``std::equal_to`` + instead of string-based type comparisons. This resolves issues when binding + types defined in the unnamed namespace. + `#4319 `_ Build system improvements: @@ -40,8 +77,17 @@ Build system improvements: * Moved the linting framework over to Ruff. `#4483 `_ +* Skip lto checks and target generation when + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. + `#4643 `_ + +* No longer inject ``-stdlib=libc++``, not needed for modern Pythons + (macOS 10.9+). + `#4639 `_ + + Version 2.10.4 (Mar 16, 2023) ----------------------------- +----------------------------- Changes: From ce9bbc0a213bfb21b77ada8de5346be5e350e3c4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 May 2023 10:03:33 -0700 Subject: [PATCH 14/78] Python 3.11+: Add `__notes__` to `error_already_set::what()` output. (#4678) * First version adding `__notes__` to `error_already_set::what()` output. * Fix trivial oversight (missing adjustment in existing test). * Minor enhancements of new code. * Re-enable `cmake --target cpptest -j 2` * Revert "Re-enable `cmake --target cpptest -j 2`" This reverts commit 60816285e9e7b95b7d33a60805888b80b2bec641. * Add general comment explaining why the `error_fetch_and_normalize` code is so unusual. --- include/pybind11/pytypes.h | 70 +++++++++++++++++++++++++++++++++----- tests/test_exceptions.py | 36 +++++++++++++++----- 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f11ed5da..f5d3f34f 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -471,13 +471,24 @@ inline const char *obj_class_name(PyObject *obj) { std::string error_string(); +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. struct error_fetch_and_normalize { - // Immediate normalization is long-established behavior (starting with - // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 - // from Sep 2016) and safest. Normalization could be deferred, but this could mask - // errors elsewhere, the performance gain is very minor in typical situations - // (usually the dominant bottleneck is EH unwinding), and the implementation here - // would be more complex. + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); if (!m_type) { @@ -492,6 +503,14 @@ struct error_fetch_and_normalize { "of the original active exception type."); } m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else // PyErr_NormalizeException() may change the exception type if there are cascading // failures. This can potentially be extremely confusing. PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); @@ -506,12 +525,12 @@ struct error_fetch_and_normalize { + " failed to obtain the name " "of the normalized active exception type."); } -#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 // This behavior runs the risk of masking errors in the error handling, but avoids a // conflict with PyPy, which relies on the normalization here to change OSError to // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). m_lazy_error_string = exc_type_name_norm; -#else +# else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -523,6 +542,7 @@ struct error_fetch_and_normalize { msg += ": " + format_value_and_trace(); pybind11_fail(msg); } +# endif #endif } @@ -558,6 +578,40 @@ struct error_fetch_and_normalize { } } } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } + } +#endif } else { result = ""; } diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index a92ab59a..ccac4536 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -317,13 +317,7 @@ def test_error_already_set_what_with_happy_exceptions( assert what == expected_what -@pytest.mark.skipif( - # Intentionally very specific: - "sys.version_info == (3, 12, 0, 'alpha', 7)", - reason="WIP: https://github.com/python/cpython/issues/102594", -) -@pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") -def test_flaky_exception_failure_point_init(): +def _test_flaky_exception_failure_point_init_before_py_3_12(): with pytest.raises(RuntimeError) as excinfo: m.error_already_set_what(FlakyException, ("failure_point_init",)) lines = str(excinfo.value).splitlines() @@ -337,7 +331,33 @@ def test_flaky_exception_failure_point_init(): # Checking the first two lines of the traceback as formatted in error_string(): assert "test_exceptions.py(" in lines[3] assert lines[3].endswith("): __init__") - assert lines[4].endswith("): test_flaky_exception_failure_point_init") + assert lines[4].endswith( + "): _test_flaky_exception_failure_point_init_before_py_3_12" + ) + + +def _test_flaky_exception_failure_point_init_py_3_12(): + # Behavior change in Python 3.12: https://github.com/python/cpython/issues/102594 + what, py_err_set_after_what = m.error_already_set_what( + FlakyException, ("failure_point_init",) + ) + assert not py_err_set_after_what + lines = what.splitlines() + assert lines[0].endswith("ValueError[WITH __notes__]: triggered_failure_point_init") + assert lines[1] == "__notes__ (len=1):" + assert "Normalization failed:" in lines[2] + assert "FlakyException" in lines[2] + + +@pytest.mark.skipif( + "env.PYPY and sys.version_info[:2] < (3, 12)", + reason="PyErr_NormalizeException Segmentation fault", +) +def test_flaky_exception_failure_point_init(): + if sys.version_info[:2] < (3, 12): + _test_flaky_exception_failure_point_init_before_py_3_12() + else: + _test_flaky_exception_failure_point_init_py_3_12() def test_flaky_exception_failure_point_str(): From 6e6bcca5b28461cd16b2af70bba837a4b24482c0 Mon Sep 17 00:00:00 2001 From: Joyce Date: Tue, 23 May 2023 14:05:25 -0300 Subject: [PATCH 15/78] Create s Security Policy (#4671) * Create SECURITY.md * Update test_files.py to include SECURITY.md file * Update MANIFEST.in to include SECURITY.md file --- MANIFEST.in | 2 +- SECURITY.md | 13 +++++++++++++ tests/extra_python_package/test_files.py | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 SECURITY.md diff --git a/MANIFEST.in b/MANIFEST.in index 31632acc..7ce83c55 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg +include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..3d74611f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e5982f96..5b9e1456 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -111,6 +111,7 @@ "MANIFEST.in", "README.rst", "PKG-INFO", + "SECURITY.md", } local_sdist_files = { From 8e1f9d5c40f74632799c8a44287d32d18a6c8630 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 23 May 2023 10:49:32 -0700 Subject: [PATCH 16/78] Add `format_descriptor<>` & `npy_format_descriptor<>` `PyObject *` specializations. (#4674) * Add `npy_format_descriptor` to enable `py::array_t` to/from-python conversions. * resolve clang-tidy warning * Use existing constructor instead of adding a static method. Thanks @Skylion007 for pointing out. * Add `format_descriptor` Trivial addition, but still in search for a meaningful test. * Add test_format_descriptor_format * Ensure the Eigen `type_caster`s do not segfault when loading arrays with dtype=object * Use `static_assert()` `!std::is_pointer<>` to replace runtime guards. * Add comments to explain how to check for ref-count bugs. (NO code changes.) * Make the "Pointer types ... are not supported" message Eigen-specific, as suggested by @Lalaland. Move to new pybind11/eigen/common.h header. * Change "format_descriptor_format" implementation as suggested by @Lalaland. Additional tests meant to ensure consistency between py::format_descriptor<>, np.array, np.format_parser turn out to be useful only to highlight long-standing inconsistencies. * resolve clang-tidy warning * Account for np.float128, np.complex256 not being available on Windows, in a future-proof way. * Fully address i|q|l ambiguity (hopefully). * Remove the new `np.format_parser()`-based test, it's much more distracting than useful. * Use bi.itemsize to disambiguate "l" or "L" * Use `py::detail::compare_buffer_info::compare()` to validate the `format_descriptor::format()` strings. * Add `buffer_info::compare` to make `detail::compare_buffer_info::compare` more visible & accessible. * silence clang-tidy warning * pytest-compatible access to np.float128, np.complex256 * Revert "pytest-compatible access to np.float128, np.complex256" This reverts commit e9a289c50fc07199806d14ded644215ab6f03afa. * Use `sizeof(long double) == sizeof(double)` instead of `std::is_same<>` * Report skipped `long double` tests. * Change the name of the new `buffer_info` member function to `item_type_is_equivalent_to`. Add comment defining "equivalent" by example. * Change `item_type_is_equivalent_to<>()` from `static` function to member function, as suggested by @Lalaland --- CMakeLists.txt | 1 + include/pybind11/buffer_info.h | 17 +++++- include/pybind11/detail/common.h | 9 +++ include/pybind11/eigen/common.h | 9 +++ include/pybind11/eigen/matrix.h | 13 +++++ include/pybind11/eigen/tensor.h | 5 ++ include/pybind11/numpy.h | 18 ++++-- tests/extra_python_package/test_files.py | 1 + tests/test_buffers.cpp | 35 ++++++++++++ tests/test_buffers.py | 57 +++++++++++++++++++ tests/test_numpy_array.cpp | 26 +++++++++ tests/test_numpy_array.py | 71 ++++++++++++++++++++++++ 12 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 include/pybind11/eigen/common.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 25a7d149..0ad74db2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,7 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h include/pybind11/eigen/matrix.h include/pybind11/eigen/tensor.h include/pybind11/embed.h diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 06120d55..b99ee8be 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector f_strides(const std::vector &shape, ssize_t return strides; } +template +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -150,6 +153,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + private: struct private_ctr_tag {}; @@ -170,9 +184,10 @@ struct buffer_info { PYBIND11_NAMESPACE_BEGIN(detail) -template +template struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 1ff0df09..c2e6f9ec 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -1025,6 +1025,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in template struct format_descriptor {}; +template +struct format_descriptor< + T, + detail::enable_if_t::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; diff --git a/include/pybind11/eigen/common.h b/include/pybind11/eigen/common.h new file mode 100644 index 00000000..24f56d15 --- /dev/null +++ b/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index bd533bed..8d4342f8 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -10,6 +10,7 @@ #pragma once #include "../numpy.h" +#include "common.h" /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: @@ -287,6 +288,8 @@ handle eigen_encapsulate(Type *src) { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using props = EigenProps; bool load(handle src, bool convert) { @@ -405,6 +408,9 @@ struct type_caster::value>> { // Base class for casting reference/map/block/etc. objects back to python. template struct eigen_map_caster { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + private: using props = EigenProps; @@ -457,6 +463,8 @@ struct type_caster< using Type = Eigen::Ref; using props = EigenProps; using Scalar = typename props::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::Map; using Array = array_t struct type_caster::value>> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + protected: using Matrix = Eigen::Matrix; @@ -632,6 +643,8 @@ struct type_caster::value>> { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using StorageIndex = remove_reference_t().outerIndexPtr())>; using Index = typename Type::Index; static constexpr bool rowMajor = Type::IsRowMajor; diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index de7dcba8..25d12bac 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -8,6 +8,7 @@ #pragma once #include "../numpy.h" +#include "common.h" #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); @@ -164,6 +165,8 @@ PYBIND11_WARNING_POP template struct type_caster::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using Helper = eigen_tensor_helper; static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); @@ -359,6 +362,8 @@ struct get_storage_pointer_type struct type_caster, typename eigen_tensor_helper>::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper>; diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 854d6e87..36077ec0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -564,6 +564,8 @@ class dtype : public object { m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType explicit dtype(int typenum) : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { if (m_ptr == nullptr) { @@ -1283,12 +1285,16 @@ struct npy_format_descriptor< public: static constexpr int value = values[detail::is_fmt_numeric::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template +struct npy_format_descriptor::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 5b9e1456..57387dd8 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -57,6 +57,7 @@ } eigen_headers = { + "include/pybind11/eigen/common.h", "include/pybind11/eigen/matrix.h", "include/pybind11/eigen/tensor.h", } diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 6b6e8cba..b5b8c355 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -7,12 +7,47 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "constructor_stats.h" #include "pybind11_tests.h" TEST_SUBMODULE(buffers, m) { + m.attr("long_double_and_double_have_same_size") = (sizeof(long double) == sizeof(double)); + + m.def("format_descriptor_format_buffer_info_equiv", + [](const std::string &cpp_name, const py::buffer &buffer) { + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *format_table = new std::map; + static auto *equiv_table + = new std::map; + if (format_table->empty()) { +#define PYBIND11_ASSIGN_HELPER(...) \ + (*format_table)[#__VA_ARGS__] = py::format_descriptor<__VA_ARGS__>::format(); \ + (*equiv_table)[#__VA_ARGS__] = &py::buffer_info::item_type_is_equivalent_to<__VA_ARGS__>; + PYBIND11_ASSIGN_HELPER(PyObject *) + PYBIND11_ASSIGN_HELPER(bool) + PYBIND11_ASSIGN_HELPER(std::int8_t) + PYBIND11_ASSIGN_HELPER(std::uint8_t) + PYBIND11_ASSIGN_HELPER(std::int16_t) + PYBIND11_ASSIGN_HELPER(std::uint16_t) + PYBIND11_ASSIGN_HELPER(std::int32_t) + PYBIND11_ASSIGN_HELPER(std::uint32_t) + PYBIND11_ASSIGN_HELPER(std::int64_t) + PYBIND11_ASSIGN_HELPER(std::uint64_t) + PYBIND11_ASSIGN_HELPER(float) + PYBIND11_ASSIGN_HELPER(double) + PYBIND11_ASSIGN_HELPER(long double) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) +#undef PYBIND11_ASSIGN_HELPER + } + return std::pair( + (*format_table)[cpp_name], (buffer.request().*((*equiv_table)[cpp_name]))()); + }); + // test_from_python / test_to_python: class Matrix { public: diff --git a/tests/test_buffers.py b/tests/test_buffers.py index eb58c467..63d9d869 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -10,6 +10,63 @@ np = pytest.importorskip("numpy") +if m.long_double_and_double_have_same_size: + # Determined by the compiler used to build the pybind11 tests + # (e.g. MSVC gets here, but MinGW might not). + np_float128 = None + np_complex256 = None +else: + # Determined by the compiler used to build numpy (e.g. MinGW). + np_float128 = getattr(np, *["float128"] * 2) + np_complex256 = getattr(np, *["complex256"] * 2) + +CPP_NAME_FORMAT_NP_DTYPE_TABLE = [ + ("PyObject *", "O", object), + ("bool", "?", np.bool_), + ("std::int8_t", "b", np.int8), + ("std::uint8_t", "B", np.uint8), + ("std::int16_t", "h", np.int16), + ("std::uint16_t", "H", np.uint16), + ("std::int32_t", "i", np.int32), + ("std::uint32_t", "I", np.uint32), + ("std::int64_t", "q", np.int64), + ("std::uint64_t", "Q", np.uint64), + ("float", "f", np.float32), + ("double", "d", np.float64), + ("long double", "g", np_float128), + ("std::complex", "Zf", np.complex64), + ("std::complex", "Zd", np.complex128), + ("std::complex", "Zg", np_complex256), +] +CPP_NAME_FORMAT_TABLE = [ + (cpp_name, format) + for cpp_name, format, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE + if np_dtype is not None +] +CPP_NAME_NP_DTYPE_TABLE = [ + (cpp_name, np_dtype) for cpp_name, _, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE +] + + +@pytest.mark.parametrize(("cpp_name", "np_dtype"), CPP_NAME_NP_DTYPE_TABLE) +def test_format_descriptor_format_buffer_info_equiv(cpp_name, np_dtype): + if np_dtype is None: + pytest.skip( + f"cpp_name=`{cpp_name}`: `long double` and `double` have same size." + ) + if isinstance(np_dtype, str): + pytest.skip(f"np.{np_dtype} does not exist.") + np_array = np.array([], dtype=np_dtype) + for other_cpp_name, expected_format in CPP_NAME_FORMAT_TABLE: + format, np_array_is_matching = m.format_descriptor_format_buffer_info_equiv( + other_cpp_name, np_array + ) + assert format == expected_format + if other_cpp_name == cpp_name: + assert np_array_is_matching + else: + assert not np_array_is_matching + def test_from_python(): with pytest.raises(RuntimeError) as excinfo: diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index b118e2c6..8c122a86 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -523,4 +523,30 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("test_fmt_desc_const_double", [](const py::array_t &) {}); sm.def("round_trip_float", [](double d) { return d; }); + + sm.def("pass_array_pyobject_ptr_return_sum_str_values", + [](const py::array_t &objs) { + std::string sum_str_values; + for (const auto &obj : objs) { + sum_str_values += py::str(obj.attr("value")); + } + return sum_str_values; + }); + + sm.def("pass_array_pyobject_ptr_return_as_list", + [](const py::array_t &objs) -> py::list { return objs; }); + + sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) { + py::size_t arr_size = py::len(objs); + py::array_t arr_from_list(static_cast(arr_size)); + PyObject **data = arr_from_list.mutable_data(); + for (py::size_t i = 0; i < arr_size; i++) { + assert(data[i] == nullptr); + data[i] = py::cast(objs[i].attr("value")); + } + return arr_from_list; + }); + + sm.def("return_array_pyobject_ptr_from_list", + [](const py::list &objs) -> py::array_t { return objs; }); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 070813d3..12e7d17d 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -595,3 +595,74 @@ def test_round_trip_float(): arr = np.zeros((), np.float64) arr[()] = 37.2 assert m.round_trip_float(arr) == 37.2 + + +# HINT: An easy and robust way (although only manual unfortunately) to check for +# ref-count leaks in the test_.*pyobject_ptr.* functions below is to +# * temporarily insert `while True:` (one-by-one), +# * run this test, and +# * run the Linux `top` command in another shell to visually monitor +# `RES` for a minute or two. +# If there is a leak, it is usually evident in seconds because the `RES` +# value increases without bounds. (Don't forget to Ctrl-C the test!) + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below: +# * Ref-count leaks will be immediately evident. +# * Sanitizers are much more likely to detect heap-use-after-free due to +# other ref-count bugs. +class PyValueHolder: + def __init__(self, value): + self.value = value + + +def WrapWithPyValueHolder(*values): + return [PyValueHolder(v) for v in values] + + +def UnwrapPyValueHolder(vhs): + return [vh.value for vh in vhs] + + +def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object) + ) + == "-3four5.0" + ) + + +def test_pass_array_pyobject_ptr_return_sum_str_values_list(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + WrapWithPyValueHolder(2, "three", -4.0) + ) + == "2three-4.0" + ) + + +def test_pass_array_pyobject_ptr_return_as_list(): + # Intentionally all temporaries, do not change. + assert UnwrapPyValueHolder( + m.pass_array_pyobject_ptr_return_as_list( + np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object) + ) + ) == [-1, "two", 3.0] + + +@pytest.mark.parametrize( + ("return_array_pyobject_ptr", "unwrap"), + [ + (m.return_array_pyobject_ptr_cpp_loop, list), + (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder), + ], +) +def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap): + # Intentionally all temporaries, do not change. + arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0)) + assert isinstance(arr_from_list, np.ndarray) + assert arr_from_list.dtype == np.dtype("O") + assert unwrap(arr_from_list) == [6, "seven", -8.0] From d0232b119f5f0cf2bf6cecacaf099c301c811ca8 Mon Sep 17 00:00:00 2001 From: "T.Yamada" Date: Thu, 25 May 2023 13:39:36 +0900 Subject: [PATCH 17/78] Use annotated for array (#4679) * use Annotated for std::array docstring * add tests * fix test * style: pre-commit fixes * fix valarray annotation * style: pre-commit fixes * refix test * add FixedSize * style: pre-commit fixes * Update test_stl.py * style: pre-commit fixes --------- Co-authored-by: Taiju Yamada Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/stl.h | 8 ++++---- tests/test_stl.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 84c51710..f39f44f7 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -273,11 +273,11 @@ struct array_caster { } PYBIND11_TYPE_CASTER(ArrayType, - const_name("List[") + value_conv::name + const_name(const_name(""), const_name("Annotated[")) + + const_name("List[") + value_conv::name + const_name("]") + const_name(const_name(""), - const_name("[") + const_name() - + const_name("]")) - + const_name("]")); + const_name(", FixedSize(") + + const_name() + const_name(")]"))); }; template diff --git a/tests/test_stl.py b/tests/test_stl.py index 2d9dcc89..8a614f8b 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -39,8 +39,11 @@ def test_array(doc): assert m.load_array(lst) assert m.load_array(tuple(lst)) - assert doc(m.cast_array) == "cast_array() -> List[int[2]]" - assert doc(m.load_array) == "load_array(arg0: List[int[2]]) -> bool" + assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]" + assert ( + doc(m.load_array) + == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool" + ) def test_valarray(doc): From 29487dee8f86c984605570c298b21c79ea3bada3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 7 Jun 2023 12:00:37 -0700 Subject: [PATCH 18/78] =?UTF-8?q?Disable=20=F0=9F=90=8D=203=20=E2=80=A2=20?= =?UTF-8?q?CentOS7=20/=20PGI=2022.9=20=E2=80=A2=20x64=20(#4691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2aba34..91144241 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -400,6 +400,7 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: + if: ${{ false }} # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690 runs-on: ubuntu-latest name: "🐍 3 • CentOS7 / PGI 22.9 • x64" container: centos:7 From 3617f3554aa5096e2148281c14be99afe9c138e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 12:57:38 -0700 Subject: [PATCH 19/78] chore(deps): update pre-commit hooks (#4689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.263 → v0.0.270](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.263...v0.0.270) - [github.com/pre-commit/mirrors-mypy: v1.2.0 → v1.3.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.2.0...v1.3.0) - [github.com/shellcheck-py/shellcheck-py: v0.9.0.2 → v0.9.0.5](https://github.com/shellcheck-py/shellcheck-py/compare/v0.9.0.2...v0.9.0.5) - [github.com/pre-commit/mirrors-clang-format: v16.0.2 → v16.0.4](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.2...v16.0.4) * Resolve new ruff error: ``` tests/test_pytypes.py:478:20: PLW0130 Duplicate value `3` in set ``` The reduction from `{3, 3}` to `{3}` never happened in pybind11 code, therefore this change is essentially a no-op for the purpose of unit-testing pybind11. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 8 ++++---- tests/test_pytypes.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aca9b3ca..bcc3f272 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.263 + rev: v0.0.270 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -100,7 +100,7 @@ repos: # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.2.0" + rev: "v1.3.0" hooks: - id: mypy args: [] @@ -128,7 +128,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.9.0.2" + rev: "v0.9.0.5" hooks: - id: shellcheck @@ -143,7 +143,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.2" + rev: "v16.0.4" hooks: - id: clang-format types_or: [c++, c, cuda] diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 946b9414..afb7a1ce 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -475,7 +475,7 @@ def test_pybind11_str_raw_str(): assert cvt({}) == "{}" assert cvt({3: 4}) == "{3: 4}" assert cvt(set()) == "set()" - assert cvt({3, 3}) == "{3}" + assert cvt({3}) == "{3}" valid_orig = "DZ" valid_utf8 = valid_orig.encode("utf-8") From c679a9209557ef0324a19f4618c556b60e251009 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 13:49:14 -0700 Subject: [PATCH 20/78] chore(deps): bump deadsnakes/action from 3.0.0 to 3.0.1 (#4687) Bumps [deadsnakes/action](https://github.com/deadsnakes/action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/deadsnakes/action/releases) - [Commits](https://github.com/deadsnakes/action/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: deadsnakes/action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91144241..b844d813 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -203,7 +203,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v3.0.0 + uses: deadsnakes/action@v3.0.1 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} From 0e43fcc75e6b7429e3511dfb44343ec05a0ab843 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 10 Jun 2023 10:14:08 -0700 Subject: [PATCH 21/78] Python 3.12b2 testing (#4695) * Uncomment `Interface test` sections (test_embed) * fix: setuptools has been removed from default installs in 3.12 --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 6 +++--- .github/workflows/upstream.yml | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b844d813..45a6738f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,9 @@ jobs: # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools if: "!(matrix.runs-on == 'windows-2022')" @@ -242,8 +244,6 @@ jobs: python -m pip install -r tests/requirements.txt - name: Configure - env: - SETUPTOOLS_USE_DISTUTILS: stdlib run: > cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 4acfbfce..d4220be2 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -1,4 +1,3 @@ - name: Upstream on: @@ -65,8 +64,8 @@ jobs: - name: Python tests C++11 run: cmake --build build11 --target pytest -j 2 - # - name: C++11 tests - # run: cmake --build build11 --target cpptest -j 2 + - name: C++11 tests + run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 run: cmake --build build11 --target test_cmake_build @@ -80,14 +79,14 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 - - name: Build + - name: Build C++17 run: cmake --build build17 -j 2 - - name: Python tests + - name: Python tests C++17 run: cmake --build build17 --target pytest - # - name: C++ tests - # run: cmake --build build17 --target cpptest + - name: C++17 tests + run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI - name: Configure (unstable ABI) @@ -105,10 +104,12 @@ jobs: - name: Python tests (unstable ABI) run: cmake --build build17max --target pytest - - name: Interface test + - name: Interface test (unstable ABI) run: cmake --build build17max --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools From 86f60a0c072e5bd5cab79233f712c321fe8deca7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 16 Jun 2023 14:09:40 -0700 Subject: [PATCH 22/78] pre-commit markdown-it-py<3 (for Python 3.7 compatibility) (#4704) * markdown-it-py dropped Python 3.7 support, but we are type checking for Python 3.7 from a newer version of Python. Keep using markdown-it-py<3 as long as we support Python 3.7. * Shuffle order of pre-commit checks for better agility. --- .pre-commit-config.yaml | 91 +++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bcc3f272..2d508276 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,48 @@ ci: exclude: ^tools/JoinPaths.cmake$ repos: + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v16.0.4" + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: "23.3.0" # Keep in sync with blacken-docs + hooks: + - id: black + +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.270 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.3.0" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: + - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. + - nox + - rich + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v4.4.0" @@ -39,12 +81,6 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: "23.3.0" # Keep in sync with blacken-docs - hooks: - - id: black - # Also code format the docs - repo: https://github.com/asottile/blacken-docs rev: "1.13.0" @@ -66,13 +102,6 @@ repos: - id: fix-ligatures - id: fix-smartquotes -# Ruff, the Python auto-correcting linter written in Rust -- repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 - hooks: - - id: ruff - args: ["--fix", "--show-fixes"] - # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks rev: "v1.10.0" @@ -81,32 +110,6 @@ repos: - id: rst-directive-colons - id: rst-inline-touching-normal - -# PyLint has native support - not always usable, but works for us -- repo: https://github.com/PyCQA/pylint - rev: "v3.0.0a6" - hooks: - - id: pylint - files: ^pybind11 - -# CMake formatting -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: "v0.6.13" - hooks: - - id: cmake-format - additional_dependencies: [pyyaml] - types: [file] - files: (\.cmake|CMakeLists.txt)(.in)?$ - -# Check static types with mypy -- repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" - hooks: - - id: mypy - args: [] - exclude: ^(tests|docs)/ - additional_dependencies: [nox, rich] - # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest rev: "0.49" @@ -141,9 +144,9 @@ repos: entry: PyBind|Numpy|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ -# Clang format the codebase automatically -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.4" +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v3.0.0a6" hooks: - - id: clang-format - types_or: [c++, c, cuda] + - id: pylint + files: ^pybind11 From 849322806cd4b3697ad1d35eedd6d0352c5f267a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 17 Jun 2023 07:02:23 -0700 Subject: [PATCH 23/78] Systematically add `PIP_BREAK_SYSTEM_PACKAGES` to all .yml files from which pip is called. (#4705) * Systematically add PIP_BREAK_SYSTEM_PACKAGES to all .yml files from which pip is called. * Try gcc:10-bullseye (because gcc:10 is broken: https://github.com/docker-library/gcc/issues/95) * bug fix (matrix did not work as hoped) --- .github/workflows/ci.yml | 17 +++++++++-------- .github/workflows/configure.yml | 1 + .github/workflows/pip.yml | 1 + .github/workflows/upstream.yml | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45a6738f..1d5d502b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ concurrency: cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy FORCE_COLOR: 3 PYTEST_TIMEOUT: 300 @@ -455,16 +456,16 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11 } - - { gcc: 7, std: 17 } - - { gcc: 8, std: 14 } - - { gcc: 8, std: 17 } - - { gcc: 10, std: 17 } - - { gcc: 11, std: 20 } - - { gcc: 12, std: 20 } + - { gcc: 7, std: 11, container_suffix: "" } + - { gcc: 7, std: 17, container_suffix: "" } + - { gcc: 8, std: 14, container_suffix: "" } + - { gcc: 8, std: 17, container_suffix: "" } + - { gcc: 10, std: 17, container_suffix: "-bullseye" } + - { gcc: 11, std: 20, container_suffix: "" } + - { gcc: 12, std: 20, container_suffix: "" } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" - container: "gcc:${{ matrix.gcc }}" + container: "gcc:${{ matrix.gcc }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 4ae22281..82a49390 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -13,6 +13,7 @@ permissions: contents: read env: + PIP_BREAK_SYSTEM_PACKAGES: 1 # For cmake: VERBOSE: 1 diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index c1feb6fe..04d95a28 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -16,6 +16,7 @@ permissions: contents: read env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy jobs: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index d4220be2..dd8a1c96 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -12,6 +12,7 @@ concurrency: cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: ":all:" # For cmake: VERBOSE: 1 From bc1bcf7c05b6097c2f0c993a776a975d7332f279 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 21 Jun 2023 13:25:10 -0400 Subject: [PATCH 24/78] chore: 3.12 + cleanup (#4713) Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 2 ++ .pre-commit-config.yaml | 1 + pybind11/setup_helpers.py | 8 ++++---- pyproject.toml | 18 ++++++++++-------- setup.cfg | 1 + setup.py | 2 +- 6 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d5d502b..1d61ad6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: - '3.9' - '3.10' - '3.11' + - '3.12' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -74,6 +75,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Setup Boost (Linux) # Can't use boost + define _ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d508276..9a829d72 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,6 +54,7 @@ repos: - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. - nox - rich + - types-setuptools # CMake formatting - repo: https://github.com/cheshirekow/cmake-format-precommit diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index cb279f27..aeeee9dc 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -66,8 +66,8 @@ from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import build_ext as _build_ext # type: ignore[assignment] + from distutils.extension import Extension as _Extension # type: ignore[assignment] import distutils.ccompiler import distutils.errors @@ -84,7 +84,7 @@ # directory into your path if it sits beside your setup.py. -class Pybind11Extension(_Extension): # type: ignore[misc] +class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you @@ -266,7 +266,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]: raise RuntimeError(msg) -class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search diff --git a/pyproject.toml b/pyproject.toml index e3655aca..59c15ea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = ["setuptools>=42", "cmake>=3.18", "ninja"] build-backend = "setuptools.build_meta" + [tool.check-manifest] ignore = [ "tests/**", @@ -15,6 +16,7 @@ ignore = [ "noxfile.py", ] + [tool.mypy] files = ["pybind11"] python_version = "3.6" @@ -24,7 +26,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true [[tool.mypy.overrides]] -module = ["ghapi.*", "setuptools.*"] +module = ["ghapi.*"] ignore_missing_imports = true @@ -55,10 +57,11 @@ messages_control.disable = [ "unused-argument", # covered by Ruff ARG ] + [tool.ruff] select = [ "E", "F", "W", # flake8 - "B", "B904", # flake8-bugbear + "B", # flake8-bugbear "I", # isort "N", # pep8-naming "ARG", # flake8-unused-arguments @@ -77,14 +80,13 @@ select = [ "YTT", # flake8-2020 ] ignore = [ - "PLR", # Design related pylint - "E501", # Line too long (Black is enough) - "PT011", # Too broad with raises in pytest - "PT004", # Fixture that doesn't return needs underscore (no, it is fine) - "SIM118",# iter(x) is not always the same as iter(x.keys()) + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118", # iter(x) is not always the same as iter(x.keys()) ] target-version = "py37" -typing-modules = ["scikit_build_core._compat.typing"] src = ["src"] unfixable = ["T20"] exclude = [] diff --git a/setup.cfg b/setup.cfg index c35dd07a..92e6c953 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython diff --git a/setup.py b/setup.py index 68573519..9fea7d35 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def get_and_replace( # Use our input files instead when making the SDist (and anything that depends # on it, like a wheel) -class SDist(setuptools.command.sdist.sdist): # type: ignore[misc] +class SDist(setuptools.command.sdist.sdist): def make_release_tree(self, base_dir: str, files: List[str]) -> None: super().make_release_tree(base_dir, files) From e10da79b6ee2554be364ef14df1c988f94df02ea Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 24 Jun 2023 12:12:35 -0700 Subject: [PATCH 25/78] Undo ci.yml gcc10 workaround after docker-library/gcc#95 was resolved. (#4717) The gcc10 workaround was introduced with PR #4705. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d61ad6a..0d640141 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -458,16 +458,16 @@ jobs: fail-fast: false matrix: include: - - { gcc: 7, std: 11, container_suffix: "" } - - { gcc: 7, std: 17, container_suffix: "" } - - { gcc: 8, std: 14, container_suffix: "" } - - { gcc: 8, std: 17, container_suffix: "" } - - { gcc: 10, std: 17, container_suffix: "-bullseye" } - - { gcc: 11, std: 20, container_suffix: "" } - - { gcc: 12, std: 20, container_suffix: "" } + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" - container: "gcc:${{ matrix.gcc }}${{ matrix.container_suffix }}" + container: "gcc:${{ matrix.gcc }}" steps: - uses: actions/checkout@v3 From 2fb3d7cbde264a0b3f921e802f287195387e8263 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 27 Jun 2023 15:08:32 -0700 Subject: [PATCH 26/78] Trivial refactoring to make the capsule API more user friendly. (#4720) * Trivial refactoring to make the capsule API more user friendly. * Use new API in production code. Thanks @Lalaland for pointing this out. --- include/pybind11/pybind11.h | 2 +- include/pybind11/pytypes.h | 51 ++++++++++++++++++++++--------------- tests/test_pytypes.cpp | 9 +++++++ tests/test_pytypes.py | 13 ++++++++++ 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 28ebc222..3bce1a01 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -508,8 +508,8 @@ class cpp_function : public function { rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), [](void *ptr) { destruct((detail::function_record *) ptr); }); - rec_capsule.set_name(detail::get_function_record_capsule_name()); guarded_strdup.release(); object scope_module; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index f5d3f34f..c93e3d3b 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1925,28 +1925,13 @@ class capsule : public object { } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast(value), nullptr, [](PyObject *o) { - // guard if destructor called while err indicator is set - error_scope error_guard; - auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr && PyErr_Occurred()) { - throw error_already_set(); - } - const char *name = get_name_in_error_scope(o); - void *ptr = PyCapsule_GetPointer(o, name); - if (ptr == nullptr) { - throw error_already_set(); - } - - if (destructor != nullptr) { - destructor(ptr); - } - }); + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { - throw error_already_set(); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { @@ -2014,6 +1999,32 @@ class capsule : public object { return name; } + + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 1028bb58..b4ee6428 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -260,6 +260,15 @@ TEST_SUBMODULE(pytypes, m) { }); }); + m.def("return_capsule_with_destructor_3", []() { + py::print("creating capsule"); + auto cap = py::capsule((void *) 1233, "oname", [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + py::print("original name: {}"_s.format(cap.name())); + return cap; + }); + m.def("return_renamed_capsule_with_destructor_2", []() { py::print("creating capsule"); auto cap = py::capsule((void *) 1234, [](void *ptr) { diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index afb7a1ce..eda7a20a 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -319,6 +319,19 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_destructor_3() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + destructing capsule: 1233 + original name: oname + """ + ) + with capture: a = m.return_renamed_capsule_with_destructor_2() del a From 5ccb9e412d8974e8eeac1b061d9077ac0bd365e1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 4 Jul 2023 08:38:14 -0400 Subject: [PATCH 27/78] chore: ruff moved to astral-sh (#4726) * chore: ruff moved to astral-sh Committed via https://github.com/asottile/all-repos * ci: bump CUDA version --- .github/workflows/ci.yml | 4 ++-- .pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d640141..39b4c237 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,8 +337,8 @@ jobs: # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest - name: "🐍 3.10 • CUDA 11.7 • Ubuntu 22.04" - container: nvidia/cuda:11.7.0-devel-ubuntu22.04 + name: "🐍 3.10 • CUDA 12.2 • Ubuntu 22.04" + container: nvidia/cuda:12.2.0-devel-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a829d72..d406624f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: - id: black # Ruff, the Python auto-correcting linter written in Rust -- repo: https://github.com/charliermarsh/ruff-pre-commit +- repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.0.270 hooks: - id: ruff From 782b6281a2aec358fb54dc1badc4b8ae1988bd41 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 9 Jul 2023 09:40:47 -0700 Subject: [PATCH 28/78] Drop PyPy 3.7 from GitHub Actions (ci.yml) (#4728) * Remove `pypy-3.7` from ci.yml * Update .github/workflows/ci.yml --------- Co-authored-by: Henry Schreiner --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39b4c237..48f7c5e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,9 +37,9 @@ jobs: - '3.10' - '3.11' - '3.12' - - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' + - 'pypy-3.10' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the From d462dd91a58116a043974b198462864a527e84ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:43:20 -0700 Subject: [PATCH 29/78] chore(deps): bump scipy from 1.8.0 to 1.10.0 in /tests (#4731) Bumps [scipy](https://github.com/scipy/scipy) from 1.8.0 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.8.0...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 04aafa8c..4ba10111 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,4 +6,4 @@ numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" pytest==7.0.0 pytest-timeout scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" -scipy==1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" +scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" From 47dc0c4bd16b7a97edb2072e93e6960647c3872f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:31:43 -0700 Subject: [PATCH 30/78] chore(deps): update pre-commit hooks (#4727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v16.0.4 → v16.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v16.0.4...v16.0.6) - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.270 → v0.0.276](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.270...v0.0.276) - [github.com/pre-commit/mirrors-mypy: v1.3.0 → v1.4.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.3.0...v1.4.1) - [github.com/asottile/blacken-docs: 1.13.0 → 1.14.0](https://github.com/asottile/blacken-docs/compare/1.13.0...1.14.0) - [github.com/codespell-project/codespell: v2.2.4 → v2.2.5](https://github.com/codespell-project/codespell/compare/v2.2.4...v2.2.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d406624f..86ac965d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: # Clang format the codebase automatically - repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v16.0.4" + rev: "v16.0.6" hooks: - id: clang-format types_or: [c++, c, cuda] @@ -38,14 +38,14 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.276 hooks: - id: ruff args: ["--fix", "--show-fixes"] # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.3.0" + rev: "v1.4.1" hooks: - id: mypy args: [] @@ -84,7 +84,7 @@ repos: # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "1.13.0" + rev: "1.14.0" hooks: - id: blacken-docs additional_dependencies: @@ -124,7 +124,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.4" + rev: "v2.2.5" hooks: - id: codespell exclude: ".supp$" From b2732c6e1a96eb97e79359c049871c41e2cd34c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 08:55:18 -0700 Subject: [PATCH 31/78] chore(deps): bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 (#4718) * chore(deps): bump pypa/gh-action-pypi-publish from 1.8.6 to 1.8.7 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.6 to 1.8.7. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.6...v1.8.7) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Apply suggestions from code review --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .github/workflows/pip.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 04d95a28..d6687b44 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -102,13 +102,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password_global }} packages-dir: global/ From 6d22dba82f1789f11a8eb2c2debbcbd4d2d8a969 Mon Sep 17 00:00:00 2001 From: Luc de Jonckheere Date: Wed, 12 Jul 2023 17:56:40 +0200 Subject: [PATCH 32/78] Warning on comparing wrapper enums with is (#4732) * Warning on comparing wrapper enums with is * backticks for quoting and link to related issue --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- docs/classes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/classes.rst b/docs/classes.rst index 52cd52da..4f2167da 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -549,3 +549,7 @@ The ``name`` property returns the name of the enum value as a unicode string. ... By default, these are omitted to conserve space. + +.. warning:: + + Contrary to Python customs, enum values from the wrappers should not be compared using ``is``, but with ``==`` (see `#1177 `_ for background). From 2e5f5c4cf8efdd62acbeb4632e8988b2148f6caf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 12 Jul 2023 13:10:24 -0500 Subject: [PATCH 33/78] fix: support CMake 3.27, drop 3.4 (#4719) * fix: support CMake 3.27, drop 3.4 Signed-off-by: Henry Schreiner * Update upgrade.rst * Update upgrade.rst * Update upgrade.rst --------- Signed-off-by: Henry Schreiner --- .github/workflows/configure.yml | 12 ++++++++---- CMakeLists.txt | 8 ++++---- docs/advanced/embedding.rst | 2 +- docs/compiling.rst | 9 ++++++--- docs/faq.rst | 3 ++- docs/upgrade.rst | 14 ++++++++++++++ tests/CMakeLists.txt | 11 ++++------- tests/test_cmake_build/CMakeLists.txt | 3 --- .../installed_embed/CMakeLists.txt | 8 ++++---- .../installed_function/CMakeLists.txt | 8 ++++---- .../installed_target/CMakeLists.txt | 8 ++++---- .../subdirectory_embed/CMakeLists.txt | 8 ++++---- .../subdirectory_function/CMakeLists.txt | 8 ++++---- .../subdirectory_target/CMakeLists.txt | 8 ++++---- tools/pybind11Common.cmake | 10 +++++++++- tools/pybind11Config.cmake.in | 4 +++- 16 files changed, 75 insertions(+), 49 deletions(-) diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 82a49390..ec7cd612 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -26,20 +26,24 @@ jobs: matrix: runs-on: [ubuntu-20.04, macos-latest, windows-latest] arch: [x64] - cmake: ["3.23"] + cmake: ["3.26"] include: - runs-on: ubuntu-20.04 arch: x64 - cmake: 3.4 + cmake: "3.5" + + - runs-on: ubuntu-20.04 + arch: x64 + cmake: "3.27" - runs-on: macos-latest arch: x64 - cmake: 3.7 + cmake: "3.7" - runs-on: windows-2019 arch: x64 # x86 compilers seem to be missing on 2019 image - cmake: 3.18 + cmake: "3.18" name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ad74db2..87ec1034 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,15 +5,15 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.22) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.22) + cmake_policy(VERSION 3.26) endif() # Avoid infinite recursion if tests include this as a subdirectory diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dd980d48..e6a1686f 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` diff --git a/docs/compiling.rst b/docs/compiling.rst index 2b543be0..1fd098be 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -241,7 +241,7 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) add_subdirectory(pybind11) @@ -261,6 +261,9 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. versionchanged:: 2.6 CMake 3.4+ is required. +.. versionchanged:: 2.11 + CMake 3.5+ is required. + Further information can be found at :doc:`cmake/index`. pybind11_add_module @@ -495,7 +498,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -553,7 +556,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) diff --git a/docs/faq.rst b/docs/faq.rst index 28498e7d..1eb00efa 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -284,7 +284,8 @@ There are three possible solutions: COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead of the old, deprecated search tools, and these modules are much better at - finding the correct Python. + finding the correct Python. If FindPythonLibs/Interp are not available + (CMake 3.27+), then this will be ignored and FindPython will be used. 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. However, you will have to use the target-based system, and do more setup yourself, because it does not know about or include things that depend on diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 6a9db2d0..b13d21f5 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,20 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.11: + +v2.11 +===== + +* The minimum version of CMake is now 3.5. A future version will likely move to + requiring something like CMake 3.15. Note that CMake 3.27 is removing the + long-deprecated support for ``FindPythonInterp`` if you set 3.27 as the + minimum or maximum supported version. To prepare for that future, CMake 3.15+ + using ``FindPython`` or setting ``PYBIND11_FINDPYTHON`` is highly recommended, + otherwise pybind11 will automatically switch to using ``FindPython`` if + ``FindPythonInterp`` is not available. + + .. _upgrade-guide-2.9: v2.9 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5574663b..80ee9c1f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,20 +5,17 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.21) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.21) + cmake_policy(VERSION 3.26) endif() -# Only needed for CMake < 3.5 support -include(CMakeParseArguments) - # Filter out items; print an optional message if any items filtered. This ignores extensions. # # Usage: diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 8bfaa386..e5aa975c 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -1,6 +1,3 @@ -# Built-in in CMake 3.5+ -include(CMakeParseArguments) - add_custom_target(test_cmake_build) function(pybind11_add_build_test name) diff --git a/tests/test_cmake_build/installed_embed/CMakeLists.txt b/tests/test_cmake_build/installed_embed/CMakeLists.txt index f7d69399..d9dcb45e 100644 --- a/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_embed CXX) diff --git a/tests/test_cmake_build/installed_function/CMakeLists.txt b/tests/test_cmake_build/installed_function/CMakeLists.txt index d7ca4db5..2f4f6427 100644 --- a/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,13 +1,13 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) project(test_installed_module CXX) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_function CXX) diff --git a/tests/test_cmake_build/installed_target/CMakeLists.txt b/tests/test_cmake_build/installed_target/CMakeLists.txt index bc5e101f..a981e236 100644 --- a/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_target CXX) diff --git a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index 58cdd7cf..f286746b 100644 --- a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_embed CXX) diff --git a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index 01557c43..275a75c0 100644 --- a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_function CXX) diff --git a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index ba82fdee..37bb2c56 100644 --- a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_target CXX) diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index e4ff1e40..308d1b70 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -163,11 +163,19 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) elseif( - PYBIND11_FINDPYTHON + _pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON OR Python_FOUND OR Python2_FOUND OR Python3_FOUND) diff --git a/tools/pybind11Config.cmake.in b/tools/pybind11Config.cmake.in index 9383e8c6..5734f437 100644 --- a/tools/pybind11Config.cmake.in +++ b/tools/pybind11Config.cmake.in @@ -63,7 +63,9 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). New FindPython mode ^^^^^^^^^^^^^^^^^^^ From b33d06f61543878a81a433ed2d472b61d3fca0ca Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 12 Jul 2023 16:20:08 -0400 Subject: [PATCH 34/78] bugfix: fixes a test suite bug in the __new__ example (#4698) * bugfix: fixes a test suite bug in the __new__ example * See https://github.com/pybind/pybind11/pull/4698#discussion_r1227107682 --------- Co-authored-by: Ralf W. Grosse-Kunstleve --- tests/test_class.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_class.cpp b/tests/test_class.cpp index ca925917..7241bc88 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -85,7 +85,7 @@ TEST_SUBMODULE(class_, m) { .def_static("new_instance", &NoConstructor::new_instance, "Return an instance"); py::class_(m, "NoConstructorNew") - .def(py::init([](const NoConstructorNew &self) { return self; })) // Need a NOOP __init__ + .def(py::init([]() { return nullptr; })) // Need a NOOP __init__ .def_static("__new__", [](const py::object &) { return NoConstructorNew::new_instance(); }); From 1a917f1852eb7819b671fc3fa862840f4c491a07 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 14 Jul 2023 08:20:38 -0700 Subject: [PATCH 35/78] docs: preparation for v2.11.0 release (#4744) * Remove .dev1 from version number. * Update Changelog (starting from `nox -s make_changelog` output) * Miscellaneous minor fixes from proofreading in GitHub web view. * docs: minor changelog updates Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner Co-authored-by: Henry Schreiner --- docs/changelog.rst | 47 +++++++++++++++++++++++++++----- docs/release.rst | 10 ++++--- include/pybind11/detail/common.h | 4 +-- pybind11/_version.py | 2 +- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 32f6606a..7bb115b9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,15 +10,15 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. -Version 2.11.0 (June 2, 2023) +Version 2.11.0 (July 14, 2023) ----------------------------- New features: -* ``pybind11::detail::is_move_constructible`` can now be specialized for cases - in which ``std::is_move_constructible`` does not work as needed. This is - very similar to the long-established - ``pybind11::detail::is_copy_constructible``. +* The newly added ``pybind11::detail::is_move_constructible`` trait can be + specialized for cases in which ``std::is_move_constructible`` does not work + as needed. This is very similar to the long-established + ``pybind11::detail::is_copy_constructible``. `#4631 `_ * Introduce ``recursive_container_traits``. @@ -30,6 +30,19 @@ New features: with ``pybind/pybind11.h``). `#4601 `_ +* ``format_descriptor<>`` & ``npy_format_descriptor<>`` ``PyObject *`` + specializations were added. The latter enables ``py::array_t`` + to/from-python conversions. + `#4674 `_ + +* ``buffer_info`` gained an ``item_type_is_equivalent_to()`` member + function. + `#4674 `_ + +* The ``capsule`` API gained a user-friendly constructor + (``py::capsule(ptr, "name", dtor)``). + `#4720 `_ + Changes: * ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & @@ -46,6 +59,12 @@ Changes: sizes slightly (~1.5%) but the error messages are much more informative. `#4463 `_ +* The docstring generation for the ``std::array``-list caster was fixed. + Previously, signatures included the size of the list in a non-standard, + non-spec compliant way. The new format conforms to PEP 593. + **Tooling for processing the docstrings may need to be updated accordingly.** + `#4679 `_ + * Setter return values (which are inaccessible for all practical purposes) are no longer converted to Python (only to be discarded). `#4621 `_ @@ -58,7 +77,7 @@ Changes: signatures on C++17 and higher. `#4587 `_ -* Compatibility with Python 3.12 (alpha). Note that the minimum pybind11 +* Compatibility with Python 3.12 (beta). Note that the minimum pybind11 ABI version for Python 3.12 is version 5. (The default ABI version for Python versions up to and including 3.11 is still version 4.). `#4570 `_ @@ -69,15 +88,23 @@ Changes: types defined in the unnamed namespace. `#4319 `_ +* Python exception ``__notes__`` (introduced with Python 3.11) are now added to + the ``error_already_set::what()`` output. + `#4678 `_ + Build system improvements: +* CMake 3.27 support was added, CMake 3.4 support was dropped. + FindPython will be used if ``FindPythonInterp`` is not present. + `#4719 `_ + * Update clang-tidy to 15 in CI. `#4387 `_ * Moved the linting framework over to Ruff. `#4483 `_ -* Skip lto checks and target generation when +* Skip ``lto`` checks and target generation when ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. `#4643 `_ @@ -85,6 +112,12 @@ Build system improvements: (macOS 10.9+). `#4639 `_ +* PyPy 3.10 support was added, PyPy 3.7 support was dropped. + `#4728 `_ + +* Testing with Python 3.12 beta releases was added. + `#4713 `_ + Version 2.10.4 (Mar 16, 2023) ----------------------------- diff --git a/docs/release.rst b/docs/release.rst index e761cdf7..4950c3b8 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -33,10 +33,12 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use - Run ``nox -s tests_packaging`` to ensure this was done correctly. - Ensure that all the information in ``setup.cfg`` is up-to-date, like supported Python versions. - - Add release date in ``docs/changelog.rst``. - - Check to make sure - `needs-changelog `_ - issues are entered in the changelog (clear the label when done). + - Add release date in ``docs/changelog.rst`` and integrate the output of + ``nox -s make_changelog``. + - Note that the ``make_changelog`` command inspects + `needs changelog `_. + - Manually clear the ``needs changelog`` labels using the GitHub web + interface (very easy: start by clicking the link above). - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it fails due to a known flake issue, either ignore or restart CI.) - Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index c2e6f9ec..27c426db 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -11,11 +11,11 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_PATCH 0 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020B0000 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 1cb51fc5..1c310e0c 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.11.0" version_info = tuple(_to_int(s) for s in __version__.split(".")) From e85696e80eb660584d5fffcb2b397eca7943f7cc Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 14 Jul 2023 14:53:58 -0700 Subject: [PATCH 36/78] Post release version bump (#4747) * Boilerplate version number update after v2.11.0 release. * Small updates to release.rst --- docs/changelog.rst | 6 ++++++ docs/release.rst | 11 +++++++---- include/pybind11/detail/common.h | 6 +++--- pybind11/_version.py | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7bb115b9..bb034d1f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,12 @@ Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. +IN DEVELOPMENT +-------------- + +Changes will be summarized here periodically. + + Version 2.11.0 (July 14, 2023) ----------------------------- diff --git a/docs/release.rst b/docs/release.rst index 4950c3b8..4d390a07 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -41,7 +41,7 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use interface (very easy: start by clicking the link above). - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it fails due to a known flake issue, either ignore or restart CI.) -- Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version +- Add a release branch if this is a new MINOR version, or update the existing release branch if it is a patch version - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` - Update branch: ``git checkout vX.Y``, ``git merge ``, ``git push`` - Update tags (optional; if you skip this, the GitHub release makes a @@ -50,7 +50,9 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use - ``git push --tags``. - Update stable - ``git checkout stable`` - - ``git merge master`` + - ``git merge -X theirs vX.Y.Z`` + - ``git diff vX.Y.Z`` + - Carefully review and reconcile any diffs. There should be none. - ``git push`` - Make a GitHub release (this shows up in the UI, sends new release notifications to users watching releases, and also uploads PyPI packages). @@ -69,9 +71,10 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use - Make sure you are on master, not somewhere else: ``git checkout master`` - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to ``0.dev1`` and increment MINOR). - - Update ``_version.py`` to match + - Update ``pybind11/_version.py`` to match - Run ``nox -s tests_packaging`` to ensure this was done correctly. - - Add a spot for in-development updates in ``docs/changelog.rst``. + - If the release was a new MINOR version, add a new `IN DEVELOPMENT` + section in ``docs/changelog.rst``. - ``git add``, ``git commit``, ``git push`` If a version branch is updated, remember to set PATCH to ``1.dev1``. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 27c426db..61f4b3e0 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0 +#define PYBIND11_VERSION_MINOR 12 +#define PYBIND11_VERSION_PATCH 0.dev1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B0000 +#define PYBIND11_VERSION_HEX 0x020C00D1 // Define some generic pybind11 helper macros for warning management. // diff --git a/pybind11/_version.py b/pybind11/_version.py index 1c310e0c..7934b7f7 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0" +__version__ = "2.12.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) From 0620d716382a5619124d8f9d7a6187825a70f018 Mon Sep 17 00:00:00 2001 From: bzaar Date: Sat, 15 Jul 2023 17:58:54 +0100 Subject: [PATCH 37/78] Update README.rst - Add missing comma in the list of acknowlegements (#4750) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3c75edb5..80213a40 100644 --- a/README.rst +++ b/README.rst @@ -135,7 +135,7 @@ This project was created by `Wenzel Jakob `_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz Miąsko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. From ec1b57c50c72a7c2ef4aa3b14d7723918da4235e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sat, 15 Jul 2023 12:55:00 -0700 Subject: [PATCH 38/78] Disable `PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF` generally for PyPy (not just PyPy Windows). (#4751) --- include/pybind11/detail/common.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 61f4b3e0..ad11b960 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -324,10 +324,7 @@ PYBIND11_WARNING_POP #endif // See description of PR #4246: -#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) \ - && !(defined(PYPY_VERSION) \ - && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the \ - process (see PR #4268) */ \ +#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(PYPY_VERSION) \ && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) # define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif From 99131a85b613aa95a7aa33d0fb1b131f0d1622e8 Mon Sep 17 00:00:00 2001 From: Ethan Steinberg Date: Mon, 17 Jul 2023 00:05:52 -0700 Subject: [PATCH 39/78] Provide `PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF` as an option (#4753) * Remove GIL checks * Update common.h * Add flag * style: pre-commit fixes * Update pytypes.h * style: pre-commit fixes * Update common.h * style: pre-commit fixes * Update pytypes.h * style: pre-commit fixes * Update common.h * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/detail/common.h | 4 ++-- include/pybind11/pytypes.h | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index ad11b960..7ebc01c5 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -324,8 +324,8 @@ PYBIND11_WARNING_POP #endif // See description of PR #4246: -#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(PYPY_VERSION) \ - && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ + && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) # define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index c93e3d3b..64aad634 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -303,7 +303,12 @@ class handle : public detail::object_api { stderr, "%s is being called while the GIL is either not held or invalid. Please see " "https://pybind11.readthedocs.io/en/stable/advanced/" - "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" + "If you are convinced there is no bug in your code, you can #define " + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF" + "to disable this check. In that case you have to ensure this #define is consistently " + "used for all translation units linked into a given pybind11 extension, otherwise " + "there will be ODR violations.", function_name.c_str()); fflush(stderr); if (Py_TYPE(m_ptr)->tp_name != nullptr) { From 8d08dc64ca300342852ceaa7b1d65fe9f69dab06 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 17 Jul 2023 12:07:42 -0700 Subject: [PATCH 40/78] Copy v2.11.1 changelog section as released. (#4755) --- docs/changelog.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bb034d1f..8e5c4965 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,21 @@ IN DEVELOPMENT Changes will be summarized here periodically. +Version 2.11.1 (July 17, 2023) +----------------------------- + +Changes: + +* ``PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF`` is now provided as an option + for disabling the default-on ``PyGILState_Check()``'s in + ``pybind11::handle``'s ``inc_ref()`` & ``dec_ref()``. + `#4753 `_ + +* ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF`` was disabled for PyPy in general + (not just PyPy Windows). + `#4751 `_ + + Version 2.11.0 (July 14, 2023) ----------------------------- From f3e0602802c7840992c97f4960515777cad6a5c7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 23 Jul 2023 11:10:10 -0700 Subject: [PATCH 41/78] Add command to check for vX.Y.Z tag vs pybind11/_version.py consistency. (#4757) * Add command to check for vX.Y.Z tag vs pybind11/_version.py consistency. Piggy-backing hints for converting changelog to release message. * Simpler way of double-checking version number in sources vs. git tag * Even simpler. * Fix rst rendering (hopefully) and remove stray dots. * [ci skip] Trying more to make GitHub rst renderer show this nicely. * [ci skip] Fix up RST rendering issues. Lesson learned: This is NOT GOOD: ``` - Bullet nesting level 1. - Bullet nesting level 2. ``` This is BETTER: ``` - Bullet nesting level 1. - Bullet nesting level 2. ``` Also consistently adding empty lines between bullet points, to make the .rst file easier to read. Also piggy-backing a few very minor enhancements. --- docs/release.rst | 131 +++++++++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 45 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 4d390a07..20b53a35 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -15,8 +15,8 @@ For example: For beta, ``PYBIND11_VERSION_PATCH`` should be ``Z.b1``. RC's can be ``Z.rc1``. Always include the dot (even though PEP 440 allows it to be dropped). For a -final release, this must be a simple integer. There is also a HEX version of -the version just below. +final release, this must be a simple integer. There is also +``PYBIND11_VERSION_HEX`` just below that needs to be updated. To release a new version of pybind11: @@ -26,56 +26,93 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use ``pipx install nox`` or ``brew install nox`` (Unix). - Update the version number - - Update ``PYBIND11_VERSION_MAJOR`` etc. in - ``include/pybind11/detail/common.h``. PATCH should be a simple integer. - - Update the version HEX just below, as well. - - Update ``pybind11/_version.py`` (match above) - - Run ``nox -s tests_packaging`` to ensure this was done correctly. - - Ensure that all the information in ``setup.cfg`` is up-to-date, like - supported Python versions. - - Add release date in ``docs/changelog.rst`` and integrate the output of - ``nox -s make_changelog``. - - Note that the ``make_changelog`` command inspects - `needs changelog `_. - - Manually clear the ``needs changelog`` labels using the GitHub web - interface (very easy: start by clicking the link above). - - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it - fails due to a known flake issue, either ignore or restart CI.) -- Add a release branch if this is a new MINOR version, or update the existing release branch if it is a patch version - - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` - - Update branch: ``git checkout vX.Y``, ``git merge ``, ``git push`` + + - Update ``PYBIND11_VERSION_MAJOR`` etc. in + ``include/pybind11/detail/common.h``. PATCH should be a simple integer. + + - Update ``PYBIND11_VERSION_HEX`` just below as well. + + - Update ``pybind11/_version.py`` (match above). + + - Run ``nox -s tests_packaging`` to ensure this was done correctly. + + - Ensure that all the information in ``setup.cfg`` is up-to-date, like + supported Python versions. + + - Add release date in ``docs/changelog.rst`` and integrate the output of + ``nox -s make_changelog``. + + - Note that the ``make_changelog`` command inspects + `needs changelog `_. + + - Manually clear the ``needs changelog`` labels using the GitHub web + interface (very easy: start by clicking the link above). + + - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it + fails due to a known flake issue, either ignore or restart CI.) + +- Add a release branch if this is a new MINOR version, or update the existing + release branch if it is a patch version + + - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` + + - Update branch: ``git checkout vX.Y``, ``git merge ``, ``git push`` + - Update tags (optional; if you skip this, the GitHub release makes a - non-annotated tag for you) - - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``. - - ``git push --tags``. + non-annotated tag for you) + + - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'`` + + - ``grep ^__version__ pybind11/_version.py`` + + - Last-minute consistency check: same as tag? + + - ``git push --tags`` + - Update stable - - ``git checkout stable`` - - ``git merge -X theirs vX.Y.Z`` - - ``git diff vX.Y.Z`` - - Carefully review and reconcile any diffs. There should be none. - - ``git push`` + + - ``git checkout stable`` + + - ``git merge -X theirs vX.Y.Z`` + + - ``git diff vX.Y.Z`` + + - Carefully review and reconcile any diffs. There should be none. + + - ``git push`` + - Make a GitHub release (this shows up in the UI, sends new release notifications to users watching releases, and also uploads PyPI packages). (Note: if you do not use an existing tag, this creates a new lightweight tag for you, so you could skip the above step.) - - GUI method: Under `releases `_ - click "Draft a new release" on the far right, fill in the tag name - (if you didn't tag above, it will be made here), fill in a release name - like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog - into the description (usually ``cat docs/changelog.rst | pandoc -f rst -t gfm``). - Check "pre-release" if this is a beta/RC. - - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` - If this is a pre-release, add ``-p``. + + - GUI method: Under `releases `_ + click "Draft a new release" on the far right, fill in the tag name + (if you didn't tag above, it will be made here), fill in a release name + like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog + into the description. You can use ``cat docs/changelog.rst | pandoc -f rst -t gfm``, + then manually remove line breaks and strip links to PRs and issues, + e.g. to a bare ``#1234``, without the surrounding ``<...>_`` hyperlink markup. + Check "pre-release" if this is a beta/RC. + + - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` + If this is a pre-release, add ``-p``. - Get back to work - - Make sure you are on master, not somewhere else: ``git checkout master`` - - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to - ``0.dev1`` and increment MINOR). - - Update ``pybind11/_version.py`` to match - - Run ``nox -s tests_packaging`` to ensure this was done correctly. - - If the release was a new MINOR version, add a new `IN DEVELOPMENT` - section in ``docs/changelog.rst``. - - ``git add``, ``git commit``, ``git push`` + + - Make sure you are on master, not somewhere else: ``git checkout master`` + + - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to + ``0.dev1`` and increment MINOR). + + - Update ``pybind11/_version.py`` to match. + + - Run ``nox -s tests_packaging`` to ensure this was done correctly. + + - If the release was a new MINOR version, add a new ``IN DEVELOPMENT`` + section in ``docs/changelog.rst``. + + - ``git add``, ``git commit``, ``git push`` If a version branch is updated, remember to set PATCH to ``1.dev1``. @@ -92,7 +129,11 @@ merge it if there are no issues. Manual packaging ^^^^^^^^^^^^^^^^ -If you need to manually upload releases, you can download the releases from the job artifacts and upload them with twine. You can also make the files locally (not recommended in general, as your local directory is more likely to be "dirty" and SDists love picking up random unrelated/hidden files); this is the procedure: +If you need to manually upload releases, you can download the releases from +the job artifacts and upload them with twine. You can also make the files +locally (not recommended in general, as your local directory is more likely +to be "dirty" and SDists love picking up random unrelated/hidden files); +this is the procedure: .. code-block:: bash From 17b614303f2176398ba3bbcec3c237628447bde5 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 3 Aug 2023 13:36:29 -0700 Subject: [PATCH 42/78] clang 17 compatibility fixes (#4767) * Copy clang 17 compatibility fixes from PR #4762 to a separate PR. * Add gcc:13 C++20 * Add silkeh/clang:16-bullseye C++20 --- .github/workflows/ci.yml | 4 ++++ include/pybind11/cast.h | 10 +++++++++- include/pybind11/pytypes.h | 10 +++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48f7c5e9..46e88132 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -302,6 +302,9 @@ jobs: - clang: 15 std: 20 container_suffix: "-bullseye" + - clang: 16 + std: 20 + container_suffix: "-bullseye" name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" @@ -465,6 +468,7 @@ jobs: - { gcc: 10, std: 17 } - { gcc: 11, std: 20 } - { gcc: 12, std: 20 } + - { gcc: 13, std: 20 } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" container: "gcc:${{ matrix.gcc }}" diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index db393411..b3c8ebe1 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1377,7 +1377,15 @@ inline namespace literals { /** \rst String literal version of `arg` \endrst */ -constexpr arg operator"" _a(const char *name, size_t) { return arg(name); } +constexpr arg +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +operator"" _a // gcc 4.8.5 insists on having a space (hard error). +#else +operator""_a // clang 17 generates a deprecation warning if there is a space. +#endif + (const char *name, size_t) { + return arg(name); +} } // namespace literals PYBIND11_NAMESPACE_BEGIN(detail) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 64aad634..580a4658 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1612,7 +1612,15 @@ inline namespace literals { /** \rst String literal version of `str` \endrst */ -inline str operator"" _s(const char *s, size_t size) { return {s, size}; } +inline str +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +operator"" _s // gcc 4.8.5 insists on having a space (hard error). +#else +operator""_s // clang 17 generates a deprecation warning if there is a space. +#endif + (const char *s, size_t size) { + return {s, size}; +} } // namespace literals /// \addtogroup pytypes From 413e6328dc0879179f1477421aaec03b700fd121 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 22:11:25 -0700 Subject: [PATCH 43/78] chore(deps): update pre-commit hooks (#4770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.3.0 → 23.7.0](https://github.com/psf/black/compare/23.3.0...23.7.0) - [github.com/astral-sh/ruff-pre-commit: v0.0.276 → v0.0.281](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.276...v0.0.281) - [github.com/asottile/blacken-docs: 1.14.0 → 1.15.0](https://github.com/asottile/blacken-docs/compare/1.14.0...1.15.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86ac965d..c7f59634 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,13 +32,13 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black - rev: "23.3.0" # Keep in sync with blacken-docs + rev: "23.7.0" # Keep in sync with blacken-docs hooks: - id: black # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.276 + rev: v0.0.281 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -84,7 +84,7 @@ repos: # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "1.14.0" + rev: "1.15.0" hooks: - id: blacken-docs additional_dependencies: From 9ad7e827a2e78516321cde6976ec9b26101f2b02 Mon Sep 17 00:00:00 2001 From: "Keto D. Zhang" Date: Thu, 3 Aug 2023 22:14:23 -0700 Subject: [PATCH 44/78] docs: Remove upper bound on pybind11 in example pyproject.toml for setuptools (#4774) * docs: Remove upper bound on pybind11 in example pyproject.toml for setuptools * Update docs/compiling.rst --------- Co-authored-by: Henry Schreiner --- docs/compiling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 1fd098be..9d53904c 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -143,7 +143,7 @@ Your ``pyproject.toml`` file will likely look something like this: .. code-block:: toml [build-system] - requires = ["setuptools>=42", "wheel", "pybind11~=2.6.1"] + requires = ["setuptools>=42", "pybind11>=2.6.1"] build-backend = "setuptools.build_meta" .. note:: From f8703154ece6ddc098db6a91c3e2f30ceacbe6b4 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Fri, 4 Aug 2023 01:48:57 -0400 Subject: [PATCH 45/78] Provide better type hints for a variety of generic types (#4259) * Provide better type hints for a variety of generic types * Makes better documentation * tuple, dict, list, set, function * Move to py::typing * style: pre-commit fixes * Update copyright line with correct year and actual author. The author information was copy-pasted from the git log output. --------- Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CMakeLists.txt | 3 +- docs/advanced/misc.rst | 29 +++++++ include/pybind11/typing.h | 97 ++++++++++++++++++++++++ tests/extra_python_package/test_files.py | 1 + tests/test_pytypes.cpp | 10 +++ tests/test_pytypes.py | 35 +++++++++ 6 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 include/pybind11/typing.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 87ec1034..15fa7990 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,7 +141,8 @@ set(PYBIND11_HEADERS include/pybind11/stl.h include/pybind11/stl_bind.h include/pybind11/stl/filesystem.h - include/pybind11/type_caster_pyobject_ptr.h) + include/pybind11/type_caster_pyobject_ptr.h + include/pybind11/typing.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 805ec838..ddd7f393 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -398,3 +398,32 @@ before they are used as a parameter or return type of a function: pyFoo.def(py::init()); pyBar.def(py::init()); } + +Setting inner type hints in docstrings +====================================== + +When you use pybind11 wrappers for ``list``, ``dict``, and other generic python +types, the docstring will just display the generic type. You can convey the +inner types in the docstring by using a special 'typed' version of the generic +type. + +.. code-block:: cpp + + PYBIND11_MODULE(example, m) { + m.def("pass_list_of_str", [](py::typing::List arg) { + // arg can be used just like py::list + )); + } + +The resulting docstring will be ``pass_list_of_str(arg0: list[str]) -> None``. + +The following special types are available in ``pybind11/typing.h``: + +* ``py::Tuple`` +* ``py::Dict`` +* ``py::List`` +* ``py::Set`` +* ``py::Callable`` + +.. warning:: Just like in python, these are merely hints. They don't actually + enforce the types of their contents at runtime or compile time. diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h new file mode 100644 index 00000000..74fd82ea --- /dev/null +++ b/include/pybind11/typing.h @@ -0,0 +1,97 @@ +/* + pybind11/typing.h: Convenience wrapper classes for basic Python types + with more explicit annotations. + + Copyright (c) 2023 Dustin Spicuzza + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(typing) + +/* + The following types can be used to direct pybind11-generated docstrings + to have have more explicit types (e.g., `list[str]` instead of `list`). + Just use these in place of existing types. + + There is no additional enforcement of types at runtime. +*/ + +template +class Tuple : public tuple { + using tuple::tuple; +}; + +template +class Dict : public dict { + using dict::dict; +}; + +template +class List : public list { + using list::list; +}; + +template +class Set : public set { + using set::set; +}; + +template +class Callable; + +template +class Callable : public function { + using function::function; +}; + +PYBIND11_NAMESPACE_END(typing) + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct handle_type_name> { + static constexpr auto name + = const_name("tuple[") + concat(make_caster::name...) + const_name("]"); +}; + +template <> +struct handle_type_name> { + // PEP 484 specifies this syntax for an empty tuple + static constexpr auto name = const_name("tuple[()]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("dict[") + make_caster::name + const_name(", ") + + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + using retval_type = conditional_t::value, void_type, Return>; + static constexpr auto name = const_name("Callable[[") + concat(make_caster::name...) + + const_name("], ") + make_caster::name + + const_name("]"); +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 57387dd8..e3d881c0 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -44,6 +44,7 @@ "include/pybind11/stl.h", "include/pybind11/stl_bind.h", "include/pybind11/type_caster_pyobject_ptr.h", + "include/pybind11/typing.h", } detail_headers = { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index b4ee6428..cb2f76c0 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +#include + #include "pybind11_tests.h" #include @@ -820,4 +822,12 @@ TEST_SUBMODULE(pytypes, m) { a >>= b; return a; }); + + m.def("annotate_tuple_float_str", [](const py::typing::Tuple &) {}); + m.def("annotate_tuple_empty", [](const py::typing::Tuple<> &) {}); + m.def("annotate_dict_str_int", [](const py::typing::Dict &) {}); + m.def("annotate_list_int", [](const py::typing::List &) {}); + m.def("annotate_set_str", [](const py::typing::Set &) {}); + m.def("annotate_fn", + [](const py::typing::Callable, py::str)> &) {}); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index eda7a20a..e88a9328 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -896,3 +896,38 @@ def test_inplace_lshift(a, b): def test_inplace_rshift(a, b): expected = a >> b assert m.inplace_rshift(a, b) == expected + + +def test_tuple_nonempty_annotations(doc): + assert ( + doc(m.annotate_tuple_float_str) + == "annotate_tuple_float_str(arg0: tuple[float, str]) -> None" + ) + + +def test_tuple_empty_annotations(doc): + assert ( + doc(m.annotate_tuple_empty) == "annotate_tuple_empty(arg0: tuple[()]) -> None" + ) + + +def test_dict_annotations(doc): + assert ( + doc(m.annotate_dict_str_int) + == "annotate_dict_str_int(arg0: dict[str, int]) -> None" + ) + + +def test_list_annotations(doc): + assert doc(m.annotate_list_int) == "annotate_list_int(arg0: list[int]) -> None" + + +def test_set_annotations(doc): + assert doc(m.annotate_set_str) == "annotate_set_str(arg0: set[str]) -> None" + + +def test_fn_annotations(doc): + assert ( + doc(m.annotate_fn) + == "annotate_fn(arg0: Callable[[list[str], str], int]) -> None" + ) From 824dc27a0195b538a7cb94411bb0dd1b783c5172 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 4 Aug 2023 11:17:33 -0700 Subject: [PATCH 46/78] CI: Reenable an NVHPC Test (#4764) Update from CentOS to Ubuntu and to a recent version of NVHPC (former: PGI). Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 53 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46e88132..ee8bdd87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -404,54 +404,55 @@ jobs: # run: cmake --build build --target test_cmake_build - # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds - centos-nvhpc7: - if: ${{ false }} # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690 - runs-on: ubuntu-latest - name: "🐍 3 • CentOS7 / PGI 22.9 • x64" - container: centos:7 + # Testing on Ubuntu + NVHPC (previous PGI) compilers, which seems to require more workarounds + ubuntu-nvhpc7: + runs-on: ubuntu-20.04 + name: "🐍 3 • NVHPC 23.5 • C++17 • x64" + env: + # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND + DEBIAN_FRONTEND: 'noninteractive' steps: - uses: actions/checkout@v3 - - name: Add Python 3 and a few requirements - run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils + - name: Add NVHPC Repo + run: | + echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ + sudo tee /etc/apt/sources.list.d/nvhpc.list - - name: Install NVidia HPC SDK - run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 + - name: Install 🐍 3 & NVHPC + run: | + sudo apt-get update -y && \ + sudo apt-get install -y cmake environment-modules git python3-dev python3-pip python3-numpy && \ + sudo apt-get install -y --no-install-recommends nvhpc-23-5 && \ + sudo rm -rf /var/lib/apt/lists/* + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade pytest - # On CentOS 7, we have to filter a few tests (compiler internal error) - # and allow deeper template recursion (not needed on CentOS 8 with a newer - # standard library). On some systems, you many need further workarounds: + # On some systems, you many need further workarounds: # https://github.com/pybind/pybind11/pull/2475 - name: Configure shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 - cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ - -DCMAKE_CXX_STANDARD=11 \ + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/23.5 + cmake -S . -B build -DDOWNLOAD_CATCH=ON \ + -DCMAKE_CXX_STANDARD=17 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" - # Building before installing Pip should produce a warning but not an error - name: Build - run: cmake3 --build build -j 2 --verbose - - - name: Install CMake with pip - run: | - python3 -m pip install --upgrade pip - python3 -m pip install pytest + run: cmake --build build -j 2 --verbose - name: Python tests - run: cmake3 --build build --target pytest + run: cmake --build build --target pytest - name: C++ tests - run: cmake3 --build build --target cpptest + run: cmake --build build --target cpptest - name: Interface test - run: cmake3 --build build --target test_cmake_build + run: cmake --build build --target test_cmake_build # Testing on GCC using the GCC docker images (only recent images supported) From 690a115d84f2b1217eb43d5f3c35d7be3052d8df Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 7 Aug 2023 20:48:20 -0700 Subject: [PATCH 47/78] Add `py::set_error()`, use in updated `py::exception<>` documentation (#4772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Copy clang 17 compatibility fixes from PR #4762 to a separate PR. * static py::exception<> -> static py::handle * Add `py::set_error()` but also try the suggestion of @malfet (https://github.com/pytorch/pytorch/pull/106401#pullrequestreview-1559961407). * clang 17 compatibility fixes (#4767) * Copy clang 17 compatibility fixes from PR #4762 to a separate PR. * Add gcc:13 C++20 * Add silkeh/clang:16-bullseye C++20 * chore(deps): update pre-commit hooks (#4770) updates: - [github.com/psf/black: 23.3.0 → 23.7.0](https://github.com/psf/black/compare/23.3.0...23.7.0) - [github.com/astral-sh/ruff-pre-commit: v0.0.276 → v0.0.281](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.276...v0.0.281) - [github.com/asottile/blacken-docs: 1.14.0 → 1.15.0](https://github.com/asottile/blacken-docs/compare/1.14.0...1.15.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * docs: Remove upper bound on pybind11 in example pyproject.toml for setuptools (#4774) * docs: Remove upper bound on pybind11 in example pyproject.toml for setuptools * Update docs/compiling.rst --------- Co-authored-by: Henry Schreiner * Provide better type hints for a variety of generic types (#4259) * Provide better type hints for a variety of generic types * Makes better documentation * tuple, dict, list, set, function * Move to py::typing * style: pre-commit fixes * Update copyright line with correct year and actual author. The author information was copy-pasted from the git log output. --------- Co-authored-by: Ralf W. Grosse-Kunstleve Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Use `py::set_error()` everywhere possible (only one special case, in common.h). Overload `py::set_error(py::handle, py::handle)`. Change back to `static py::handle exc = ... .release();` Deprecate `py::exception<>::operator()` * Add `PYBIND11_WARNING_DISABLE` for INTEL and MSVC (and sort alphabetically). * `PYBIND11_WARNING_DISABLE_INTEL(10441)` does not work. For ICC only, falling back to the recommended `py::set_error()` to keep the testing simple. It is troublesome to add `--diag-disable=10441` specifically for test_exceptions.cpp, even that is non-ideal because it covers the entire file, not just the one line we need it for, and the value of exercising the trivial deprecated `operator()` on this one extra platform is practically zero. * Fix silly oversight. * NVHPC 23.5.0 generates deprecation warnings. They are currently not treated as errors, but falling back to using `py::set_error()` to not have to deal with that distraction. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Keto D. Zhang Co-authored-by: Henry Schreiner Co-authored-by: Dustin Spicuzza --- docs/advanced/exceptions.rst | 23 +++---- include/pybind11/detail/class.h | 6 +- include/pybind11/detail/common.h | 2 +- include/pybind11/detail/internals.h | 2 +- include/pybind11/detail/type_caster_base.h | 2 +- include/pybind11/numpy.h | 4 +- include/pybind11/pybind11.h | 22 +++---- include/pybind11/pytypes.h | 8 +++ ...s_module_interleaved_error_already_set.cpp | 4 +- tests/pybind11_cross_module_tests.cpp | 8 +-- tests/test_exceptions.cpp | 60 +++++++++++++++---- tests/test_exceptions.h | 2 +- tests/test_exceptions.py | 10 +++- tests/test_pytypes.cpp | 2 +- tests/test_type_caster_pyobject_ptr.cpp | 4 +- 15 files changed, 105 insertions(+), 54 deletions(-) diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 53981dc0..616e2d77 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -127,8 +127,7 @@ before a global translator is tried. Inside the translator, ``std::rethrow_exception`` should be used within a try block to re-throw the exception. One or more catch clauses to catch the appropriate exceptions should then be used with each clause using -``PyErr_SetString`` to set a Python exception or ``ex(string)`` to set -the python exception to a custom exception type (see below). +``py::set_error()`` (see below). To declare a custom Python exception type, declare a ``py::exception`` variable and use this in the associated exception translator (note: it is often useful @@ -142,14 +141,17 @@ standard python RuntimeError: .. code-block:: cpp - static py::exception exc(m, "MyCustomError"); + // This is a static object, so we must leak the Python reference: + // It is undefined when the destructor will run, possibly only after the + // Python interpreter is finalized already. + static py::handle exc = py::exception(m, "MyCustomError").release(); py::register_exception_translator([](std::exception_ptr p) { try { if (p) std::rethrow_exception(p); } catch (const MyCustomException &e) { - exc(e.what()); + py::set_error(exc, e.what()); } catch (const OtherException &e) { - PyErr_SetString(PyExc_RuntimeError, e.what()); + py::set_error(PyExc_RuntimeError, e.what()); } }); @@ -168,8 +170,7 @@ section. .. note:: - Call either ``PyErr_SetString`` or a custom exception's call - operator (``exc(string)``) for every exception caught in a custom exception + Call ``py::set_error()`` for every exception caught in a custom exception translator. Failure to do so will cause Python to crash with ``SystemError: error return without exception set``. @@ -200,7 +201,7 @@ If module1 has the following translator: try { if (p) std::rethrow_exception(p); } catch (const std::invalid_argument &e) { - PyErr_SetString("module1 handled this") + py::set_error(PyExc_ArgumentError, "module1 handled this"); } } @@ -212,7 +213,7 @@ and module2 has the following similar translator: try { if (p) std::rethrow_exception(p); } catch (const std::invalid_argument &e) { - PyErr_SetString("module2 handled this") + py::set_error(PyExc_ArgumentError, "module2 handled this"); } } @@ -312,11 +313,11 @@ error protocol, which is outlined here. After calling the Python C API, if Python returns an error, ``throw py::error_already_set();``, which allows pybind11 to deal with the exception and pass it back to the Python interpreter. This includes calls to -the error setting functions such as ``PyErr_SetString``. +the error setting functions such as ``py::set_error()``. .. code-block:: cpp - PyErr_SetString(PyExc_TypeError, "C API type error demo"); + py::set_error(PyExc_TypeError, "C API type error demo"); throw py::error_already_set(); // But it would be easier to simply... diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index bc2b40c5..3ece0643 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -375,7 +375,7 @@ extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, extern "C" inline int pybind11_object_init(PyObject *self, PyObject *, PyObject *) { PyTypeObject *type = Py_TYPE(self); std::string msg = get_fully_qualified_tp_name(type) + ": No constructor defined!"; - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return -1; } @@ -579,7 +579,7 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla if (view) { view->obj = nullptr; } - PyErr_SetString(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); + set_error(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); return -1; } std::memset(view, 0, sizeof(Py_buffer)); @@ -587,7 +587,7 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { delete info; // view->obj = nullptr; // Was just memset to 0, so not necessary - PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); + set_error(PyExc_BufferError, "Writable buffer requested for readonly storage"); return -1; } view->obj = obj; diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 7ebc01c5..e79f7693 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -399,7 +399,7 @@ PYBIND11_WARNING_POP return nullptr; \ } \ catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ + ::pybind11::set_error(PyExc_ImportError, e.what()); \ return nullptr; \ } diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index aaa7f868..228dd764 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -352,7 +352,7 @@ inline bool raise_err(PyObject *exc_type, const char *msg) { raise_from(exc_type, msg); return true; } - PyErr_SetString(exc_type, msg); + set_error(exc_type, msg); return false; } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 16387506..fc5f1c88 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -786,7 +786,7 @@ class type_caster_generic { std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); detail::clean_type_id(tname); std::string msg = "Unregistered type : " + tname; - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return {nullptr, nullptr}; } diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 36077ec0..02f65d74 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1008,7 +1008,7 @@ class array : public buffer { /// Create array from any object -- always returns a new reference static PyObject *raw_array(PyObject *ptr, int ExtraFlags = 0) { if (ptr == nullptr) { - PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); + set_error(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); return nullptr; } return detail::npy_api::get().PyArray_FromAny_( @@ -1155,7 +1155,7 @@ class array_t : public array { /// Create array from any object -- always returns a new reference static PyObject *raw_array_t(PyObject *ptr) { if (ptr == nullptr) { - PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); + set_error(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); return nullptr; } return detail::npy_api::get().PyArray_FromAny_(ptr, diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3bce1a01..76edf7f1 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -694,9 +694,8 @@ class cpp_function : public function { if (overloads->is_constructor) { if (!parent || !PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { - PyErr_SetString( - PyExc_TypeError, - "__init__(self, ...) called with invalid or missing `self` argument"); + set_error(PyExc_TypeError, + "__init__(self, ...) called with invalid or missing `self` argument"); return nullptr; } @@ -1007,7 +1006,7 @@ class cpp_function : public function { A translator may choose to do one of the following: - - catch the exception and call PyErr_SetString or PyErr_SetObject + - catch the exception and call py::set_error() to set a standard (or custom) Python exception, or - do nothing and let the exception fall through to the next translator, or - delegate translation to the next translator by throwing a new type of exception. @@ -1023,8 +1022,7 @@ class cpp_function : public function { return nullptr; } - PyErr_SetString(PyExc_SystemError, - "Exception escaped from default exception translator!"); + set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); return nullptr; } @@ -1125,7 +1123,7 @@ class cpp_function : public function { raise_from(PyExc_TypeError, msg.c_str()); return nullptr; } - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return nullptr; } if (!result) { @@ -1138,7 +1136,7 @@ class cpp_function : public function { raise_from(PyExc_TypeError, msg.c_str()); return nullptr; } - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return nullptr; } if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { @@ -2528,7 +2526,7 @@ inline void register_local_exception_translator(ExceptionTranslator &&translator /** * Wrapper to generate a new Python exception type. * - * This should only be used with PyErr_SetString for now. + * This should only be used with py::set_error() for now. * It is not (yet) possible to use as a py::base. * Template type argument is reserved for future use. */ @@ -2549,7 +2547,9 @@ class exception : public object { } // Sets the current python exception to this exception object with the given message - void operator()(const char *message) { PyErr_SetString(m_ptr, message); } + PYBIND11_DEPRECATED("Please use py::set_error() instead " + "(https://github.com/pybind/pybind11/pull/4772)") + void operator()(const char *message) const { set_error(*this, message); } }; PYBIND11_NAMESPACE_BEGIN(detail) @@ -2581,7 +2581,7 @@ register_exception_impl(handle scope, const char *name, handle base, bool isLoca try { std::rethrow_exception(p); } catch (const CppException &e) { - detail::get_exception_object()(e.what()); + set_error(detail::get_exception_object(), e.what()); } }); return ex; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 580a4658..fa2d2498 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -334,6 +334,14 @@ class handle : public detail::object_api { #endif }; +inline void set_error(const handle &type, const char *message) { + PyErr_SetString(type.ptr(), message); +} + +inline void set_error(const handle &type, const handle &value) { + PyErr_SetObject(type.ptr(), value.ptr()); +} + /** \rst Holds a reference to a Python object (with reference counting) diff --git a/tests/cross_module_interleaved_error_already_set.cpp b/tests/cross_module_interleaved_error_already_set.cpp index fdd9939e..3493a7e6 100644 --- a/tests/cross_module_interleaved_error_already_set.cpp +++ b/tests/cross_module_interleaved_error_already_set.cpp @@ -16,12 +16,12 @@ namespace { namespace py = pybind11; void interleaved_error_already_set() { - PyErr_SetString(PyExc_RuntimeError, "1st error."); + py::set_error(PyExc_RuntimeError, "1st error."); try { throw py::error_already_set(); } catch (const py::error_already_set &) { // The 2nd error could be conditional in a real application. - PyErr_SetString(PyExc_RuntimeError, "2nd error."); + py::set_error(PyExc_RuntimeError, "2nd error."); } // Here the 1st error is destroyed before the 2nd error is fetched. // The error_already_set dtor triggers a pybind11::detail::get_internals() // call via pybind11::gil_scoped_acquire. diff --git a/tests/pybind11_cross_module_tests.cpp b/tests/pybind11_cross_module_tests.cpp index 9379f3f2..ad68e9a5 100644 --- a/tests/pybind11_cross_module_tests.cpp +++ b/tests/pybind11_cross_module_tests.cpp @@ -31,11 +31,11 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { // test_exceptions.py py::register_local_exception(m, "LocalSimpleException"); m.def("raise_runtime_error", []() { - PyErr_SetString(PyExc_RuntimeError, "My runtime error"); + py::set_error(PyExc_RuntimeError, "My runtime error"); throw py::error_already_set(); }); m.def("raise_value_error", []() { - PyErr_SetString(PyExc_ValueError, "My value error"); + py::set_error(PyExc_ValueError, "My value error"); throw py::error_already_set(); }); m.def("throw_pybind_value_error", []() { throw py::value_error("pybind11 value error"); }); @@ -49,7 +49,7 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { std::rethrow_exception(p); } } catch (const shared_exception &e) { - PyErr_SetString(PyExc_KeyError, e.what()); + py::set_error(PyExc_KeyError, e.what()); } }); @@ -60,7 +60,7 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { std::rethrow_exception(p); } } catch (const LocalException &e) { - PyErr_SetString(PyExc_KeyError, e.what()); + py::set_error(PyExc_KeyError, e.what()); } }); diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index 854c7e6f..f6d9c215 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -25,6 +25,10 @@ class MyException : public std::exception { std::string message = ""; }; +class MyExceptionUseDeprecatedOperatorCall : public MyException { + using MyException::MyException; +}; + // A type that should be translated to a standard Python exception class MyException2 : public std::exception { public: @@ -109,8 +113,8 @@ TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); - // make a new custom exception and use it as a translation target - static py::exception ex(m, "MyException"); + // PLEASE KEEP IN SYNC with docs/advanced/exceptions.rst + static py::handle ex = py::exception(m, "MyException").release(); py::register_exception_translator([](std::exception_ptr p) { try { if (p) { @@ -118,7 +122,32 @@ TEST_SUBMODULE(exceptions, m) { } } catch (const MyException &e) { // Set MyException as the active python error - ex(e.what()); + py::set_error(ex, e.what()); + } + }); + + // Same as above, but using the deprecated `py::exception<>::operator()` + // We want to be sure it still works, until it's removed. + static const auto *const exd = new py::exception( + m, "MyExceptionUseDeprecatedOperatorCall"); + py::register_exception_translator([](std::exception_ptr p) { + try { + if (p) { + std::rethrow_exception(p); + } + } catch (const MyExceptionUseDeprecatedOperatorCall &e) { +#if defined(__INTEL_COMPILER) || defined(__NVCOMPILER) + // It is not worth the trouble dealing with warning suppressions for these compilers. + // Falling back to the recommended approach to keep the test code simple. + py::set_error(*exd, e.what()); +#else + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated-declarations") + PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated-declarations") + PYBIND11_WARNING_DISABLE_MSVC(4996) + (*exd)(e.what()); + PYBIND11_WARNING_POP +#endif } }); @@ -132,7 +161,7 @@ TEST_SUBMODULE(exceptions, m) { } } catch (const MyException2 &e) { // Translate this exception to a standard RuntimeError - PyErr_SetString(PyExc_RuntimeError, e.what()); + py::set_error(PyExc_RuntimeError, e.what()); } }); @@ -162,11 +191,16 @@ TEST_SUBMODULE(exceptions, m) { std::rethrow_exception(p); } } catch (const MyException6 &e) { - PyErr_SetString(PyExc_RuntimeError, e.what()); + py::set_error(PyExc_RuntimeError, e.what()); } }); - m.def("throws1", []() { throw MyException("this error should go to a custom type"); }); + m.def("throws1", + []() { throw MyException("this error should go to py::exception"); }); + m.def("throws1d", []() { + throw MyExceptionUseDeprecatedOperatorCall( + "this error should go to py::exception"); + }); m.def("throws2", []() { throw MyException2("this error should go to a standard Python exception"); }); m.def("throws3", []() { throw MyException3("this error cannot be translated"); }); @@ -222,7 +256,7 @@ TEST_SUBMODULE(exceptions, m) { m.def("throw_already_set", [](bool err) { if (err) { - PyErr_SetString(PyExc_ValueError, "foo"); + py::set_error(PyExc_ValueError, "foo"); } try { throw py::error_already_set(); @@ -238,7 +272,7 @@ TEST_SUBMODULE(exceptions, m) { } PyErr_Clear(); if (err) { - PyErr_SetString(PyExc_ValueError, "foo"); + py::set_error(PyExc_ValueError, "foo"); } throw py::error_already_set(); }); @@ -247,7 +281,7 @@ TEST_SUBMODULE(exceptions, m) { bool retval = false; try { PythonCallInDestructor set_dict_in_destructor(d); - PyErr_SetString(PyExc_ValueError, "foo"); + py::set_error(PyExc_ValueError, "foo"); throw py::error_already_set(); } catch (const py::error_already_set &) { retval = true; @@ -282,14 +316,14 @@ TEST_SUBMODULE(exceptions, m) { m.def("throw_should_be_translated_to_key_error", []() { throw shared_exception(); }); m.def("raise_from", []() { - PyErr_SetString(PyExc_ValueError, "inner"); + py::set_error(PyExc_ValueError, "inner"); py::raise_from(PyExc_ValueError, "outer"); throw py::error_already_set(); }); m.def("raise_from_already_set", []() { try { - PyErr_SetString(PyExc_ValueError, "inner"); + py::set_error(PyExc_ValueError, "inner"); throw py::error_already_set(); } catch (py::error_already_set &e) { py::raise_from(e, PyExc_ValueError, "outer"); @@ -306,7 +340,7 @@ TEST_SUBMODULE(exceptions, m) { }); m.def("error_already_set_what", [](const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); + py::set_error(exc_type, exc_value); std::string what = py::error_already_set().what(); bool py_err_set_after_what = (PyErr_Occurred() != nullptr); PyErr_Clear(); @@ -321,7 +355,7 @@ TEST_SUBMODULE(exceptions, m) { }); m.def("test_error_already_set_double_restore", [](bool dry_run) { - PyErr_SetString(PyExc_ValueError, "Random error."); + py::set_error(PyExc_ValueError, "Random error."); py::error_already_set e; e.restore(); PyErr_Clear(); diff --git a/tests/test_exceptions.h b/tests/test_exceptions.h index 03684b89..2eaa3d3d 100644 --- a/tests/test_exceptions.h +++ b/tests/test_exceptions.h @@ -9,5 +9,5 @@ class PYBIND11_EXPORT_EXCEPTION shared_exception : public pybind11::builtin_exce public: using builtin_exception::builtin_exception; explicit shared_exception() : shared_exception("") {} - void set_error() const override { PyErr_SetString(PyExc_RuntimeError, what()); } + void set_error() const override { py::set_error(PyExc_RuntimeError, what()); } }; diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index ccac4536..5d8e6292 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -139,7 +139,15 @@ def test_custom(msg): # Can we catch a MyException? with pytest.raises(m.MyException) as excinfo: m.throws1() - assert msg(excinfo.value) == "this error should go to a custom type" + assert msg(excinfo.value) == "this error should go to py::exception" + + # Can we catch a MyExceptionUseDeprecatedOperatorCall? + with pytest.raises(m.MyExceptionUseDeprecatedOperatorCall) as excinfo: + m.throws1d() + assert ( + msg(excinfo.value) + == "this error should go to py::exception" + ) # Can we translate to standard Python exceptions? with pytest.raises(RuntimeError) as excinfo: diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index cb2f76c0..b03f5624 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -25,7 +25,7 @@ PyObject *conv(PyObject *o) { ret = PyFloat_FromDouble(v); } } else { - PyErr_SetString(PyExc_TypeError, "Unexpected type"); + py::set_error(PyExc_TypeError, "Unexpected type"); } return ret; } diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp index 1667ea12..8069f7dc 100644 --- a/tests/test_type_caster_pyobject_ptr.cpp +++ b/tests/test_type_caster_pyobject_ptr.cpp @@ -78,14 +78,14 @@ TEST_SUBMODULE(type_caster_pyobject_ptr, m) { m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) { if (set_error) { - PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling."); + py::set_error(PyExc_RuntimeError, "Reflective of healthy error handling."); } PyObject *ptr = nullptr; py::cast(ptr); }); m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() { - PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling."); + py::set_error(PyExc_RuntimeError, "Reflective of unhealthy error handling."); py::cast(Py_None); }); From 4bf60c609a8af3868b813a27072d51c080c3e175 Mon Sep 17 00:00:00 2001 From: Pieter P Date: Tue, 8 Aug 2023 05:58:30 +0200 Subject: [PATCH 48/78] Disable strip when build type is unset (#4454) (#4780) --- tools/pybind11NewTools.cmake | 12 +++++++----- tools/pybind11Tools.cmake | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 7d7424a7..3a35a738 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -233,11 +233,13 @@ function(pybind11_add_module target_name) endif() endif() - # Use case-insensitive comparison to match the result of $ - string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) - # Strip unnecessary sections of the binary on Linux/macOS - pybind11_strip(${target_name}) + if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) + # Strip unnecessary sections of the binary on Linux/macOS + pybind11_strip(${target_name}) + endif() endif() if(MSVC) diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 66ad00a4..67e710bd 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -212,10 +212,12 @@ function(pybind11_add_module target_name) endif() endif() - # Use case-insensitive comparison to match the result of $ - string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) - pybind11_strip(${target_name}) + if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) + pybind11_strip(${target_name}) + endif() endif() if(MSVC) From 9039e6ac425e5d77dbef9b281124679e7dc421c4 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 10 Aug 2023 15:17:56 -0400 Subject: [PATCH 49/78] chore: use 2x faster black mirror (#4784) Committed via https://github.com/asottile/all-repos --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7f59634..4102ee3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: types_or: [c++, c, cuda] # Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black +- repo: https://github.com/psf/black-pre-commit-mirror rev: "23.7.0" # Keep in sync with blacken-docs hooks: - id: black From add281a2da7a6eae78c99d20cc3034b0b5381524 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 14 Aug 2023 21:46:17 -0700 Subject: [PATCH 50/78] =?UTF-8?q?Migrate=20to=20readthedocs=20configuratio?= =?UTF-8?q?n=20file=20v2=C2=B6=20(#4789)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Copy of recommded readthedocs configuration file v2 * [ci skip] It is now requirements (not requirements_file) --- .readthedocs.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index c9c61617..2625521e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,3 +1,15 @@ +# https://blog.readthedocs.com/migrate-configuration-v2/ + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + python: - version: 3 -requirements_file: docs/requirements.txt + install: + - requirements: docs/requirements.txt From 80bcd21fb75b1dd95c0d9b095bb1657c3588d85e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 15 Aug 2023 07:02:54 -0700 Subject: [PATCH 51/78] [ci skip] Adopt nanobind config. (#4792) --- .readthedocs.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 2625521e..a2b802f7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,6 +4,8 @@ version: 2 build: os: ubuntu-22.04 + apt_packages: + - librsvg2-bin tools: python: "3.11" @@ -13,3 +15,6 @@ sphinx: python: install: - requirements: docs/requirements.txt + +formats: + - pdf From f47ff3280e30c275c0a633293bf08d23d3728668 Mon Sep 17 00:00:00 2001 From: Kenji Date: Tue, 15 Aug 2023 10:09:13 -0400 Subject: [PATCH 52/78] Fix grammar in functions.rst (#4791) The previous sentence had an extra "a" before "several", which isn't right. --- docs/advanced/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index 69e3d8a1..372934b0 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -16,7 +16,7 @@ lifetime of objects managed by them. This can lead to issues when creating bindings for functions that return a non-trivial type. Just by looking at the type information, it is not clear whether Python should take charge of the returned value and eventually free its resources, or if this is handled on the -C++ side. For this reason, pybind11 provides a several *return value policy* +C++ side. For this reason, pybind11 provides several *return value policy* annotations that can be passed to the :func:`module_::def` and :func:`class_::def` functions. The default policy is :enum:`return_value_policy::automatic`. From b9359ceadbf4d4b22509b684eea107d055b77901 Mon Sep 17 00:00:00 2001 From: Jean Elsner Date: Tue, 15 Aug 2023 16:48:59 +0200 Subject: [PATCH 53/78] Remove newlines from docstring signature (#4735) * Remove newlines from docstring signature * Jean/dev (#1) Replace newlines in arg values with spaces * style: pre-commit fixes * Don't use std::find_if for C++ 11 compatibility * Avoid implicit char to bool conversion * Test default arguments for line breaks * style: pre-commit fixes * Separate Eigen tests * style: pre-commit fixes * Fix merge * Try importing numpy * Avoid unreferenced variable in catch block * style: pre-commit fixes * Update squash function * Reduce try block * Additional test cases * style: pre-commit fixes * Put statement inside braces * Move string into function body * Rename repr for better readability. Make constr explicit. * Add multiline string default argument test case * style: pre-commit fixes * Add std namespace, do not modify string repr * Test for all space chars, test str repr not modified --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/pybind11.h | 41 +++++++++++++++++++++++++++- tests/test_eigen_matrix.cpp | 17 ++++++++++++ tests/test_eigen_matrix.py | 5 ++++ tests/test_kwargs_and_defaults.cpp | 44 ++++++++++++++++++++++++++++++ tests/test_kwargs_and_defaults.py | 32 ++++++++++++++++++++++ 5 files changed, 138 insertions(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 76edf7f1..db91639d 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -52,6 +52,45 @@ PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) +inline std::string replace_newlines_and_squash(const char *text) { + const char *whitespaces = " \t\n\r\f\v"; + std::string result(text); + bool previous_is_whitespace = false; + + // Do not modify string representations + char first_char = result[0]; + char last_char = result[result.size() - 1]; + if (first_char == last_char && first_char == '\'') { + return result; + } + result.clear(); + + // Replace characters in whitespaces array with spaces and squash consecutive spaces + while (*text != '\0') { + if (std::strchr(whitespaces, *text)) { + if (!previous_is_whitespace) { + result += ' '; + previous_is_whitespace = true; + } + } else { + result += *text; + previous_is_whitespace = false; + } + ++text; + } + + // Strip leading and trailing whitespaces + const size_t str_begin = result.find_first_not_of(whitespaces); + if (str_begin == std::string::npos) { + return ""; + } + + const size_t str_end = result.find_last_not_of(whitespaces); + const size_t str_range = str_end - str_begin + 1; + + return result.substr(str_begin, str_range); +} + // Apply all the extensions translators from a list // Return true if one of the translators completed without raising an exception // itself. Return of false indicates that if there are other translators @@ -424,7 +463,7 @@ class cpp_function : public function { // Write default value if available. if (!is_starred && arg_index < rec->args.size() && rec->args[arg_index].descr) { signature += " = "; - signature += rec->args[arg_index].descr; + signature += detail::replace_newlines_and_squash(rec->args[arg_index].descr); } // Separator for positional-only arguments (placed after the // argument, rather than before like * diff --git a/tests/test_eigen_matrix.cpp b/tests/test_eigen_matrix.cpp index 554cc4d7..0c003d05 100644 --- a/tests/test_eigen_matrix.cpp +++ b/tests/test_eigen_matrix.cpp @@ -330,6 +330,23 @@ TEST_SUBMODULE(eigen_matrix, m) { m.def("dense_c", [mat]() -> DenseMatrixC { return DenseMatrixC(mat); }); m.def("dense_copy_r", [](const DenseMatrixR &m) -> DenseMatrixR { return m; }); m.def("dense_copy_c", [](const DenseMatrixC &m) -> DenseMatrixC { return m; }); + // test_defaults + bool have_numpy = true; + try { + py::module_::import("numpy"); + } catch (const py::error_already_set &) { + have_numpy = false; + } + if (have_numpy) { + py::module_::import("numpy"); + Eigen::Matrix defaultMatrix = Eigen::Matrix3d::Identity(); + m.def( + "defaults_mat", [](const Eigen::Matrix3d &) {}, py::arg("mat") = defaultMatrix); + + Eigen::VectorXd defaultVector = Eigen::VectorXd::Ones(32); + m.def( + "defaults_vec", [](const Eigen::VectorXd &) {}, py::arg("vec") = defaultMatrix); + } // test_sparse, test_sparse_signature m.def("sparse_r", [mat]() -> SparseMatrixR { // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn) diff --git a/tests/test_eigen_matrix.py b/tests/test_eigen_matrix.py index b2e76740..a486c2f9 100644 --- a/tests/test_eigen_matrix.py +++ b/tests/test_eigen_matrix.py @@ -716,6 +716,11 @@ def test_dense_signature(doc): ) +def test_defaults(doc): + assert "\n" not in str(doc(m.defaults_mat)) + assert "\n" not in str(doc(m.defaults_vec)) + + def test_named_arguments(): a = np.array([[1.0, 2], [3, 4], [5, 6]]) b = np.ones((2, 1)) diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 77e72c0c..614c826d 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -42,6 +42,50 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { m.def("kw_func_udl", kw_func, "x"_a, "y"_a = 300); m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a = 0); + // test line breaks in default argument representation + struct CustomRepr { + std::string repr_string; + + explicit CustomRepr(const std::string &repr) : repr_string(repr) {} + + std::string __repr__() const { return repr_string; } + }; + + py::class_(m, "CustomRepr") + .def(py::init()) + .def("__repr__", &CustomRepr::__repr__); + + m.def( + "kw_lb_func0", + [](const CustomRepr &) {}, + py::arg("custom") = CustomRepr(" array([[A, B], [C, D]]) ")); + m.def( + "kw_lb_func1", + [](const CustomRepr &) {}, + py::arg("custom") = CustomRepr(" array([[A, B],\n[C, D]]) ")); + m.def( + "kw_lb_func2", + [](const CustomRepr &) {}, + py::arg("custom") = CustomRepr("\v\n array([[A, B], [C, D]])")); + m.def( + "kw_lb_func3", + [](const CustomRepr &) {}, + py::arg("custom") = CustomRepr("array([[A, B], [C, D]]) \f\n")); + m.def( + "kw_lb_func4", + [](const CustomRepr &) {}, + py::arg("custom") = CustomRepr("array([[A, B],\n\f\n[C, D]])")); + m.def( + "kw_lb_func5", + [](const CustomRepr &) {}, + py::arg("custom") = CustomRepr("array([[A, B],\r [C, D]])")); + m.def( + "kw_lb_func6", [](const CustomRepr &) {}, py::arg("custom") = CustomRepr(" \v\t ")); + m.def( + "kw_lb_func7", + [](const std::string &) {}, + py::arg("str_arg") = "First line.\n Second line."); + // test_args_and_kwargs m.def("args_function", [](py::args args) -> py::tuple { PYBIND11_WARNING_PUSH diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 7174726f..b5770019 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -23,6 +23,38 @@ def test_function_signatures(doc): doc(m.KWClass.foo1) == "foo1(self: m.kwargs_and_defaults.KWClass, x: int, y: float) -> None" ) + assert ( + doc(m.kw_lb_func0) + == "kw_lb_func0(custom: m.kwargs_and_defaults.CustomRepr = array([[A, B], [C, D]])) -> None" + ) + assert ( + doc(m.kw_lb_func1) + == "kw_lb_func1(custom: m.kwargs_and_defaults.CustomRepr = array([[A, B], [C, D]])) -> None" + ) + assert ( + doc(m.kw_lb_func2) + == "kw_lb_func2(custom: m.kwargs_and_defaults.CustomRepr = array([[A, B], [C, D]])) -> None" + ) + assert ( + doc(m.kw_lb_func3) + == "kw_lb_func3(custom: m.kwargs_and_defaults.CustomRepr = array([[A, B], [C, D]])) -> None" + ) + assert ( + doc(m.kw_lb_func4) + == "kw_lb_func4(custom: m.kwargs_and_defaults.CustomRepr = array([[A, B], [C, D]])) -> None" + ) + assert ( + doc(m.kw_lb_func5) + == "kw_lb_func5(custom: m.kwargs_and_defaults.CustomRepr = array([[A, B], [C, D]])) -> None" + ) + assert ( + doc(m.kw_lb_func6) + == "kw_lb_func6(custom: m.kwargs_and_defaults.CustomRepr = ) -> None" + ) + assert ( + doc(m.kw_lb_func7) + == "kw_lb_func7(str_arg: str = 'First line.\\n Second line.') -> None" + ) def test_named_arguments(): From 76b885811096421eff6ccd612de93e328ff3461f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20W=C3=BCrtz?= Date: Wed, 23 Aug 2023 18:49:35 +0200 Subject: [PATCH 54/78] fix: Different MSVC versions may be ABI incompatible, guard with _MSC_VER (#2898) (#4779) --- include/pybind11/detail/internals.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 228dd764..a9136fa8 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -291,9 +291,12 @@ struct type_info { #endif /// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. +/// On MSVC, changes in _MSC_VER may indicate ABI incompatibility (#2898). #ifndef PYBIND11_BUILD_ABI # if defined(__GXX_ABI_VERSION) # define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) +# elif defined(_MSC_VER) +# define PYBIND11_BUILD_ABI "_mscver" PYBIND11_TOSTRING(_MSC_VER) # else # define PYBIND11_BUILD_ABI "" # endif From 1adac5a5b104be5f0a8caae75667640df125594a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 30 Aug 2023 10:05:24 -0700 Subject: [PATCH 55/78] `PYBIND11_INTERNALS_VERSION` bump for MSVC, piggy-backed on PR #4779. See comments there. (#4819) --- include/pybind11/detail/internals.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index a9136fa8..e2c94ea9 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -34,8 +34,9 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# if PY_VERSION_HEX >= 0x030C0000 +# if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER) // Version bump for Python 3.12+, before first 3.12 beta release. +// Version bump for MSVC piggy-backed on PR #4779. See comments there. # define PYBIND11_INTERNALS_VERSION 5 # else # define PYBIND11_INTERNALS_VERSION 4 From e705fb5f2764446a6ae7b4ae4f51dcef8c107006 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Thu, 31 Aug 2023 06:20:46 +0900 Subject: [PATCH 56/78] Fix enum's `__str__` docstring (#4827) * fix: Enum __str__ function name * tests: Test enum.__str__.__doc__ --- include/pybind11/pybind11.h | 2 +- tests/test_enum.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index db91639d..e6c9ad0a 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2014,7 +2014,7 @@ struct enum_base { object type_name = type::handle_of(arg).attr("__name__"); return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg)); }, - name("name"), + name("__str__"), is_method(m_base)); if (options::show_enum_members_docstring()) { diff --git a/tests/test_enum.py b/tests/test_enum.py index 4e85d29c..b97b0fa5 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -264,3 +264,8 @@ def test_docstring_signatures(): for attr in enum_type.__dict__.values(): # Issue #2623/PR #2637: Add argument names to enum_ methods assert "arg0" not in (attr.__doc__ or "") + + +def test_str_signature(): + for enum_type in [m.ScopedEnum, m.UnscopedEnum]: + assert enum_type.__str__.__doc__.startswith("__str__") From db412e6e8648a5687d73ef4cf28738d1e7f0e53f Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Thu, 31 Aug 2023 14:43:01 +0900 Subject: [PATCH 57/78] fix: Render `py::function` as `Callable` (#4829) * fix: Render `py::function` as `Callable` * style: pre-commit fixes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/pybind11/cast.h | 4 ++++ tests/test_callbacks.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index b3c8ebe1..8a4e2e64 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -898,6 +898,10 @@ struct handle_type_name { static constexpr auto name = const_name("float"); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("Callable"); +}; +template <> struct handle_type_name { static constexpr auto name = const_name("None"); }; diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 4a652f53..86c76745 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -216,3 +216,10 @@ def test_custom_func(): def test_custom_func2(): assert m.custom_function2(3) == 27 assert m.roundtrip(m.custom_function2)(3) == 27 + + +def test_callback_docstring(): + assert ( + m.test_tuple_unpacking.__doc__.strip() + == "test_tuple_unpacking(arg0: Callable) -> object" + ) From c9638a1927ca6cfe731bd27eeef885cfc26a762b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 1 Sep 2023 11:31:22 -0700 Subject: [PATCH 58/78] Help Coverty avoid generating a false positive. (#4817) * Change variable name `it` to `curr_overl` to improve readability, add `assert()` in the only place where `curr_overl` could become `nullptr`. * Move the `assert()` to where Coverty generates a false positive. * variable name as suggested by @henryiii --- include/pybind11/pybind11.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index e6c9ad0a..0a82f8dd 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -719,7 +719,7 @@ class cpp_function : public function { /* Iterator over the list of potentially admissible overloads */ const function_record *overloads = reinterpret_cast( PyCapsule_GetPointer(self, get_function_record_capsule_name())), - *it = overloads; + *current_overload = overloads; assert(overloads != nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right @@ -757,9 +757,10 @@ class cpp_function : public function { std::vector second_pass; // However, if there are no overloads, we can just skip the no-convert pass entirely - const bool overloaded = it != nullptr && it->next != nullptr; + const bool overloaded + = current_overload != nullptr && current_overload->next != nullptr; - for (; it != nullptr; it = it->next) { + for (; current_overload != nullptr; current_overload = current_overload->next) { /* For each overload: 1. Copy all positional arguments we were given, also checking to make sure that @@ -780,7 +781,7 @@ class cpp_function : public function { a result other than PYBIND11_TRY_NEXT_OVERLOAD. */ - const function_record &func = *it; + const function_record &func = *current_overload; size_t num_args = func.nargs; // Number of positional arguments that we need if (func.has_args) { --num_args; // (but don't count py::args @@ -1018,10 +1019,10 @@ class cpp_function : public function { } if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { - // The error reporting logic below expects 'it' to be valid, as it would be - // if we'd encountered this failure in the first-pass loop. + // The error reporting logic below expects 'current_overload' to be valid, + // as it would be if we'd encountered this failure in the first-pass loop. if (!result) { - it = &call.func; + current_overload = &call.func; } break; } @@ -1168,7 +1169,8 @@ class cpp_function : public function { if (!result) { std::string msg = "Unable to convert function return value to a " "Python type! The signature was\n\t"; - msg += it->signature; + assert(current_overload != nullptr); + msg += current_overload->signature; append_note_if_missing_header_is_suspected(msg); // Attach additional error info to the exception if supported if (PyErr_Occurred()) { From 467fe27bd9f4f335bd1423d532927fa894446bec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 06:04:27 -0700 Subject: [PATCH 59/78] chore(deps): update pre-commit hooks (#4838) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.281 → v0.0.287](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.281...v0.0.287) - [github.com/pre-commit/mirrors-mypy: v1.4.1 → v1.5.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.4.1...v1.5.1) - [github.com/asottile/blacken-docs: 1.15.0 → 1.16.0](https://github.com/asottile/blacken-docs/compare/1.15.0...1.16.0) - [github.com/Lucas-C/pre-commit-hooks: v1.5.1 → v1.5.4](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.5.1...v1.5.4) - [github.com/PyCQA/pylint: v3.0.0a6 → v3.0.0a7](https://github.com/PyCQA/pylint/compare/v3.0.0a6...v3.0.0a7) * style: pre-commit fixes * Update pyproject.toml --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .pre-commit-config.yaml | 10 +++++----- docs/benchmark.py | 2 +- pyproject.toml | 2 +- tests/test_stl_binders.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4102ee3b..f1fb4ae1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,14 +38,14 @@ repos: # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.281 + rev: v0.0.287 hooks: - id: ruff args: ["--fix", "--show-fixes"] # Check static types with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.4.1" + rev: "v1.5.1" hooks: - id: mypy args: [] @@ -84,7 +84,7 @@ repos: # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "1.15.0" + rev: "1.16.0" hooks: - id: blacken-docs additional_dependencies: @@ -92,7 +92,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.5.1" + rev: "v1.5.4" hooks: - id: remove-tabs @@ -147,7 +147,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.0.0a6" + rev: "v3.0.0a7" hooks: - id: pylint files: ^pybind11 diff --git a/docs/benchmark.py b/docs/benchmark.py index 2150b6ca..fb49fd04 100644 --- a/docs/benchmark.py +++ b/docs/benchmark.py @@ -70,7 +70,7 @@ def generate_dummy_code_boost(nclasses=10): for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: print("{") - for i in range(0, 10): + for i in range(10): nclasses = 2**i with open("test.cpp", "w") as f: f.write(codegen(nclasses)) diff --git a/pyproject.toml b/pyproject.toml index 59c15ea6..ef2a8736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,5 +94,5 @@ line-length = 120 isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] [tool.ruff.per-file-ignores] -"tests/**" = ["EM", "N"] +"tests/**" = ["EM", "N", "E721"] "tests/test_call_policies.py" = ["PLC1901"] diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index e002f5b6..79923f48 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -209,7 +209,7 @@ def test_map_string_double_const(): def test_noncopyable_containers(): # std::vector vnc = m.get_vnc(5) - for i in range(0, 5): + for i in range(5): assert vnc[i].value == i + 1 for i, j in enumerate(vnc, start=1): @@ -217,7 +217,7 @@ def test_noncopyable_containers(): # std::deque dnc = m.get_dnc(5) - for i in range(0, 5): + for i in range(5): assert dnc[i].value == i + 1 i = 1 @@ -252,7 +252,7 @@ def test_noncopyable_containers(): # nested std::map nvnc = m.get_nvnc(5) for i in range(1, 6): - for j in range(0, 5): + for j in range(5): assert nvnc[i][j].value == j + 1 # Note: maps do not have .values() From 4a2f7e4681cc912d4251a74c04a96c8c5a3896fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:57:18 -0400 Subject: [PATCH 60/78] chore(deps): bump actions/checkout from 1 to 4 (#4836) * chore(deps): bump actions/checkout from 1 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 1 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v1...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update .github/workflows/ci.yml * actions/checkout@v1 for centos:7 * Fix oversight: centos:7 actually works with actions/checkout@v3 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner Co-authored-by: Ralf W. Grosse-Kunstleve --- .github/workflows/ci.yml | 40 +++++++++++++++++++-------------- .github/workflows/configure.yml | 2 +- .github/workflows/format.yml | 4 ++-- .github/workflows/pip.yml | 4 ++-- .github/workflows/upstream.yml | 2 +- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee8bdd87..9fcf0ca5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v4 @@ -205,7 +205,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) uses: deadsnakes/action@v3.0.1 @@ -310,7 +310,7 @@ jobs: container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add wget and python3 run: apt-get update && apt-get install -y python3-dev python3-numpy python3-pytest libeigen3-dev @@ -344,7 +344,7 @@ jobs: container: nvidia/cuda:12.2.0-devel-ubuntu22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND - name: Install 🐍 3 @@ -368,7 +368,7 @@ jobs: # container: centos:8 # # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # # - name: Add Python 3 and a few requirements # run: yum update -y && yum install -y git python3-devel python3-numpy python3-pytest make environment-modules @@ -413,7 +413,7 @@ jobs: # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND DEBIAN_FRONTEND: 'noninteractive' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add NVHPC Repo run: | @@ -475,7 +475,7 @@ jobs: container: "gcc:${{ matrix.gcc }}" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add Python 3 run: apt-get update; apt-get install -y python3-dev python3-numpy python3-pytest python3-pip libeigen3-dev @@ -535,7 +535,7 @@ jobs: name: "🐍 3 • ICC latest • x64" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add apt repo run: | @@ -639,7 +639,13 @@ jobs: container: "${{ matrix.container }}" steps: - - uses: actions/checkout@v3 + - name: Latest actions/checkout + uses: actions/checkout@v4 + if: matrix.container != 'centos:7' + + - name: Pin actions/checkout as required for centos:7 + uses: actions/checkout@v3 + if: matrix.container == 'centos:7' - name: Add Python 3 (RHEL 7) if: matrix.container == 'centos:7' @@ -687,7 +693,7 @@ jobs: container: i386/debian:buster steps: - - uses: actions/checkout@v1 # Required to run inside docker + - uses: actions/checkout@v1 # v1 is required to run inside docker - name: Install requirements run: | @@ -730,7 +736,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: @@ -782,7 +788,7 @@ jobs: runs-on: windows-2019 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v4 @@ -835,7 +841,7 @@ jobs: runs-on: windows-2019 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v4 @@ -883,7 +889,7 @@ jobs: runs-on: windows-2022 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v4 @@ -961,7 +967,7 @@ jobs: mingw-w64-${{matrix.env}}-boost mingw-w64-${{matrix.env}}-catch - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure C++11 # LTO leads to many undefined reference like @@ -1032,7 +1038,7 @@ jobs: run: env - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Clang uses: egor-tensin/setup-clang@v1 @@ -1101,7 +1107,7 @@ jobs: run: env - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Show Clang++ version before brew install llvm run: clang++ --version diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index ec7cd612..a49d8d25 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -49,7 +49,7 @@ jobs: runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python 3.7 uses: actions/setup-python@v4 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b8242ee5..9b2da594 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -25,7 +25,7 @@ jobs: name: Format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.x" @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest container: silkeh/clang:15-bullseye steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install requirements run: apt-get update && apt-get install -y git python3-dev python3-pytest diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index d6687b44..5bc7c4b9 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -28,7 +28,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup 🐍 3.6 uses: actions/setup-python@v4 @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup 🐍 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index dd8a1c96..5d893cd8 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -25,7 +25,7 @@ jobs: if: "contains(github.event.pull_request.labels.*.name, 'python dev')" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python 3.12 uses: actions/setup-python@v4 From c83605936b5e09ae806c40fe798d1e0b1143e272 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Thu, 7 Sep 2023 21:57:39 +0900 Subject: [PATCH 61/78] feature: Support move-only iterators in `py::make_*iterator` (#4834) * feature: Support move-only iterators in `py::make_*iterator` * fix: Missing static assertion message * fixup: Missing `explicit` in single argument constructors * fix: Simplify tests: make existing iterator move-only * fix: Missing `noexcept` --- include/pybind11/pybind11.h | 14 ++++++++++---- tests/test_sequences_and_iterators.cpp | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 0a82f8dd..5e3c5c65 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2434,7 +2434,7 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { Policy); } - return cast(state{first, last, true}); + return cast(state{std::forward(first), std::forward(last), true}); } PYBIND11_NAMESPACE_END(detail) @@ -2451,7 +2451,9 @@ iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { Iterator, Sentinel, ValueType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2467,7 +2469,9 @@ iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { Iterator, Sentinel, KeyType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2483,7 +2487,9 @@ iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { Iterator, Sentinel, ValueType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index 1de65edb..4a1d37f4 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -28,6 +28,13 @@ class NonZeroIterator { public: explicit NonZeroIterator(const T *ptr) : ptr_(ptr) {} + + // Make the iterator non-copyable and movable + NonZeroIterator(const NonZeroIterator &) = delete; + NonZeroIterator(NonZeroIterator &&) noexcept = default; + NonZeroIterator &operator=(const NonZeroIterator &) = delete; + NonZeroIterator &operator=(NonZeroIterator &&) noexcept = default; + const T &operator*() const { return *ptr_; } NonZeroIterator &operator++() { ++ptr_; @@ -78,6 +85,7 @@ class NonCopyableInt { int value_; }; using NonCopyableIntPair = std::pair; + PYBIND11_MAKE_OPAQUE(std::vector); PYBIND11_MAKE_OPAQUE(std::vector); @@ -375,6 +383,17 @@ TEST_SUBMODULE(sequences_and_iterators, m) { private: std::vector> data_; }; + + { + // #4383 : Make sure `py::make_*iterator` functions work with move-only iterators + using iterator_t = NonZeroIterator>; + + static_assert(std::is_move_assignable::value, ""); + static_assert(std::is_move_constructible::value, ""); + static_assert(!std::is_copy_assignable::value, ""); + static_assert(!std::is_copy_constructible::value, ""); + } + py::class_(m, "IntPairs") .def(py::init>>()) .def( From c9149d995c801831632b654e0e8e65f2f2f31e2b Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Wed, 13 Sep 2023 04:46:58 +0900 Subject: [PATCH 62/78] fix: Use lowercase builtin collection names (#4833) --- include/pybind11/cast.h | 2 +- include/pybind11/stl.h | 8 ++++---- tests/test_builtin_casters.py | 4 ++-- tests/test_kwargs_and_defaults.py | 2 +- tests/test_stl.py | 24 ++++++++++++------------ 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 8a4e2e64..99bf4fac 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -661,7 +661,7 @@ class tuple_caster { } static constexpr auto name - = const_name("Tuple[") + concat(make_caster::name...) + const_name("]"); + = const_name("tuple[") + concat(make_caster::name...) + const_name("]"); template using cast_op_type = type; diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index f39f44f7..6eb48599 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -100,7 +100,7 @@ struct set_caster { return s.release(); } - PYBIND11_TYPE_CASTER(type, const_name("Set[") + key_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(type, const_name("set[") + key_conv::name + const_name("]")); }; template @@ -157,7 +157,7 @@ struct map_caster { } PYBIND11_TYPE_CASTER(Type, - const_name("Dict[") + key_conv::name + const_name(", ") + value_conv::name + const_name("dict[") + key_conv::name + const_name(", ") + value_conv::name + const_name("]")); }; @@ -208,7 +208,7 @@ struct list_caster { return l.release(); } - PYBIND11_TYPE_CASTER(Type, const_name("List[") + value_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(Type, const_name("list[") + value_conv::name + const_name("]")); }; template @@ -274,7 +274,7 @@ struct array_caster { PYBIND11_TYPE_CASTER(ArrayType, const_name(const_name(""), const_name("Annotated[")) - + const_name("List[") + value_conv::name + const_name("]") + + const_name("list[") + value_conv::name + const_name("]") + const_name(const_name(""), const_name(", FixedSize(") + const_name() + const_name(")]"))); diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index b1f57bdd..dbac1cbc 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -352,7 +352,7 @@ def test_tuple(doc): assert ( doc(m.pair_passthrough) == """ - pair_passthrough(arg0: Tuple[bool, str]) -> Tuple[str, bool] + pair_passthrough(arg0: tuple[bool, str]) -> tuple[str, bool] Return a pair in reversed order """ @@ -360,7 +360,7 @@ def test_tuple(doc): assert ( doc(m.tuple_passthrough) == """ - tuple_passthrough(arg0: Tuple[bool, str, int]) -> Tuple[int, str, bool] + tuple_passthrough(arg0: tuple[bool, str, int]) -> tuple[int, str, bool] Return a triple in reversed order """ diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index b5770019..9d9738de 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -8,7 +8,7 @@ def test_function_signatures(doc): assert doc(m.kw_func1) == "kw_func1(x: int, y: int) -> str" assert doc(m.kw_func2) == "kw_func2(x: int = 100, y: int = 200) -> str" assert doc(m.kw_func3) == "kw_func3(data: str = 'Hello world!') -> None" - assert doc(m.kw_func4) == "kw_func4(myList: List[int] = [13, 17]) -> str" + assert doc(m.kw_func4) == "kw_func4(myList: list[int] = [13, 17]) -> str" assert doc(m.kw_func_udl) == "kw_func_udl(x: int, y: int = 300) -> str" assert doc(m.kw_func_udl_z) == "kw_func_udl_z(x: int, y: int = 0) -> str" assert doc(m.args_function) == "args_function(*args) -> tuple" diff --git a/tests/test_stl.py b/tests/test_stl.py index 8a614f8b..b08bd468 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -16,8 +16,8 @@ def test_vector(doc): assert m.load_bool_vector([True, False]) assert m.load_bool_vector((True, False)) - assert doc(m.cast_vector) == "cast_vector() -> List[int]" - assert doc(m.load_vector) == "load_vector(arg0: List[int]) -> bool" + assert doc(m.cast_vector) == "cast_vector() -> list[int]" + assert doc(m.load_vector) == "load_vector(arg0: list[int]) -> bool" # Test regression caused by 936: pointers to stl containers weren't castable assert m.cast_ptr_vector() == ["lvalue", "lvalue"] @@ -39,10 +39,10 @@ def test_array(doc): assert m.load_array(lst) assert m.load_array(tuple(lst)) - assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]" + assert doc(m.cast_array) == "cast_array() -> Annotated[list[int], FixedSize(2)]" assert ( doc(m.load_array) - == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool" + == "load_array(arg0: Annotated[list[int], FixedSize(2)]) -> bool" ) @@ -53,8 +53,8 @@ def test_valarray(doc): assert m.load_valarray(lst) assert m.load_valarray(tuple(lst)) - assert doc(m.cast_valarray) == "cast_valarray() -> List[int]" - assert doc(m.load_valarray) == "load_valarray(arg0: List[int]) -> bool" + assert doc(m.cast_valarray) == "cast_valarray() -> list[int]" + assert doc(m.load_valarray) == "load_valarray(arg0: list[int]) -> bool" def test_map(doc): @@ -66,8 +66,8 @@ def test_map(doc): assert "key2" in d assert m.load_map(d) - assert doc(m.cast_map) == "cast_map() -> Dict[str, str]" - assert doc(m.load_map) == "load_map(arg0: Dict[str, str]) -> bool" + assert doc(m.cast_map) == "cast_map() -> dict[str, str]" + assert doc(m.load_map) == "load_map(arg0: dict[str, str]) -> bool" def test_set(doc): @@ -78,8 +78,8 @@ def test_set(doc): assert m.load_set(s) assert m.load_set(frozenset(s)) - assert doc(m.cast_set) == "cast_set() -> Set[str]" - assert doc(m.load_set) == "load_set(arg0: Set[str]) -> bool" + assert doc(m.cast_set) == "cast_set() -> set[str]" + assert doc(m.load_set) == "load_set(arg0: set[str]) -> bool" def test_recursive_casting(): @@ -303,7 +303,7 @@ def test_stl_pass_by_pointer(msg): msg(excinfo.value) == """ stl_pass_by_pointer(): incompatible function arguments. The following argument types are supported: - 1. (v: List[int] = None) -> List[int] + 1. (v: list[int] = None) -> list[int] Invoked with: """ @@ -315,7 +315,7 @@ def test_stl_pass_by_pointer(msg): msg(excinfo.value) == """ stl_pass_by_pointer(): incompatible function arguments. The following argument types are supported: - 1. (v: List[int] = None) -> List[int] + 1. (v: list[int] = None) -> list[int] Invoked with: None """ From b4573674bc65a7842b49be45d63b582f4c15d1b4 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Wed, 13 Sep 2023 04:47:39 +0900 Subject: [PATCH 63/78] Update render for buffer sequence and handle (#4831) * fix: Add capitalize render name of `py::buffer` and `py::sequence` * fix: Render `py::handle` same way as `py::object` * tests: Fix tests `handle` -> `object` * tests: Test capitaliation of `py::sequence` and `py::buffer` * style: pre-commit fixes * fix: Render `py::object` as `Any` * Revert "fix: Render `py::object` as `Any`" This reverts commit 7861dcfabb78ac210b4c67c35a0d47fb67525a96. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- include/pybind11/cast.h | 12 ++++++++++++ tests/test_buffers.py | 7 +++++++ tests/test_factory_constructors.py | 4 ++-- tests/test_sequences_and_iterators.py | 4 ++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 99bf4fac..3c9b7c92 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -882,6 +882,10 @@ struct handle_type_name { static constexpr auto name = const_name(PYBIND11_BYTES_NAME); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("Buffer"); +}; +template <> struct handle_type_name { static constexpr auto name = const_name("int"); }; @@ -902,10 +906,18 @@ struct handle_type_name { static constexpr auto name = const_name("Callable"); }; template <> +struct handle_type_name { + static constexpr auto name = handle_type_name::name; +}; +template <> struct handle_type_name { static constexpr auto name = const_name("None"); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("Sequence"); +}; +template <> struct handle_type_name { static constexpr auto name = const_name("*args"); }; diff --git a/tests/test_buffers.py b/tests/test_buffers.py index 63d9d869..5d33625b 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -219,3 +219,10 @@ def test_ctypes_from_buffer(): assert cinfo.shape == pyinfo.shape assert cinfo.strides == pyinfo.strides assert not cinfo.readonly + + +def test_buffer_docstring(): + assert ( + m.get_buffer_info.__doc__.strip() + == "get_buffer_info(arg0: Buffer) -> pybind11_tests.buffers.buffer_info" + ) diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index 04df8026..a9004cbf 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -77,7 +77,7 @@ def test_init_factory_signature(msg): 1. m.factory_constructors.TestFactory1(arg0: m.factory_constructors.tag.unique_ptr_tag, arg1: int) 2. m.factory_constructors.TestFactory1(arg0: str) 3. m.factory_constructors.TestFactory1(arg0: m.factory_constructors.tag.pointer_tag) - 4. m.factory_constructors.TestFactory1(arg0: handle, arg1: int, arg2: handle) + 4. m.factory_constructors.TestFactory1(arg0: object, arg1: int, arg2: object) Invoked with: 'invalid', 'constructor', 'arguments' """ @@ -95,7 +95,7 @@ def test_init_factory_signature(msg): 3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None - 4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None + 4. __init__(self: m.factory_constructors.TestFactory1, arg0: object, arg1: int, arg2: object) -> None """ ) diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index dc129f2b..acbe9d89 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -171,6 +171,10 @@ def __len__(self): assert m.sequence_length("hello") == 5 +def test_sequence_doc(): + assert m.sequence_length.__doc__.strip() == "sequence_length(arg0: Sequence) -> int" + + def test_map_iterator(): sm = m.StringMap({"hi": "bye", "black": "white"}) assert sm["hi"] == "bye" From 8c7b8dd0ae74b36b7d42f77b0dd4096ebb7f4ab1 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Wed, 13 Sep 2023 04:48:27 +0900 Subject: [PATCH 64/78] fix: Missing typed variants of `iterator` and `iterable` (#4832) --- include/pybind11/typing.h | 20 ++++++++++++++++++++ tests/test_pytypes.cpp | 2 ++ tests/test_pytypes.py | 14 ++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 74fd82ea..b7b1a4e5 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -45,6 +45,16 @@ class Set : public set { using set::set; }; +template +class Iterable : public iterable { + using iterable::iterable; +}; + +template +class Iterator : public iterator { + using iterator::iterator; +}; + template class Callable; @@ -85,6 +95,16 @@ struct handle_type_name> { static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); }; +template +struct handle_type_name> { + static constexpr auto name = const_name("Iterable[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("Iterator[") + make_caster::name + const_name("]"); +}; + template struct handle_type_name> { using retval_type = conditional_t::value, void_type, Return>; diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index b03f5624..0a587680 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -828,6 +828,8 @@ TEST_SUBMODULE(pytypes, m) { m.def("annotate_dict_str_int", [](const py::typing::Dict &) {}); m.def("annotate_list_int", [](const py::typing::List &) {}); m.def("annotate_set_str", [](const py::typing::Set &) {}); + m.def("annotate_iterable_str", [](const py::typing::Iterable &) {}); + m.def("annotate_iterator_int", [](const py::typing::Iterator &) {}); m.def("annotate_fn", [](const py::typing::Callable, py::str)> &) {}); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index e88a9328..2b202731 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -926,6 +926,20 @@ def test_set_annotations(doc): assert doc(m.annotate_set_str) == "annotate_set_str(arg0: set[str]) -> None" +def test_iterable_annotations(doc): + assert ( + doc(m.annotate_iterable_str) + == "annotate_iterable_str(arg0: Iterable[str]) -> None" + ) + + +def test_iterator_annotations(doc): + assert ( + doc(m.annotate_iterator_int) + == "annotate_iterator_int(arg0: Iterator[int]) -> None" + ) + + def test_fn_annotations(doc): assert ( doc(m.annotate_fn) From d06d53694ab30b48865c8f502374243517a768f3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 14 Sep 2023 09:47:34 -0700 Subject: [PATCH 65/78] Fix small bug introduced with PR #4735 (#4845) * Bug fix: `result[0]` called if `result.empty()` * Add unit test that fails without the fix. --- include/pybind11/pybind11.h | 12 +++++++----- tests/test_kwargs_and_defaults.cpp | 2 ++ tests/test_kwargs_and_defaults.py | 4 ++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 5e3c5c65..25ca508a 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -57,11 +57,13 @@ inline std::string replace_newlines_and_squash(const char *text) { std::string result(text); bool previous_is_whitespace = false; - // Do not modify string representations - char first_char = result[0]; - char last_char = result[result.size() - 1]; - if (first_char == last_char && first_char == '\'') { - return result; + if (result.size() >= 2) { + // Do not modify string representations + char first_char = result[0]; + char last_char = result[result.size() - 1]; + if (first_char == last_char && first_char == '\'') { + return result; + } } result.clear(); diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 614c826d..9a12c42a 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -85,6 +85,8 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { "kw_lb_func7", [](const std::string &) {}, py::arg("str_arg") = "First line.\n Second line."); + m.def( + "kw_lb_func8", [](const CustomRepr &) {}, py::arg("custom") = CustomRepr("")); // test_args_and_kwargs m.def("args_function", [](py::args args) -> py::tuple { diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 9d9738de..0e3bbdb7 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -55,6 +55,10 @@ def test_function_signatures(doc): doc(m.kw_lb_func7) == "kw_lb_func7(str_arg: str = 'First line.\\n Second line.') -> None" ) + assert ( + doc(m.kw_lb_func8) + == "kw_lb_func8(custom: m.kwargs_and_defaults.CustomRepr = ) -> None" + ) def test_named_arguments(): From 4fb111bd78abe5f94a7462f1f565a4dc41b225a9 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 15 Sep 2023 17:59:30 -0400 Subject: [PATCH 66/78] fix(cmake): correctly detect FindPython policy and better warning (#4806) --- CMakeLists.txt | 10 ++++++++++ tools/FindPythonLibsNew.cmake | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 15fa7990..65ad0c22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,11 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. +# Propagate this policy (FindPythonInterp removal) so it can be detected later +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) +endif() + cmake_minimum_required(VERSION 3.5) # The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with @@ -16,6 +21,11 @@ else() cmake_policy(VERSION 3.26) endif() +if(_pybind11_cmp0148) + cmake_policy(SET CMP0148 ${_pybind11_cmp0148}) + unset(_pybind11_cmp0148) +endif() + # Avoid infinite recursion if tests include this as a subdirectory if(DEFINED PYBIND11_MASTER_PROJECT) return() diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index ce558d4e..992f9f52 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -95,6 +95,22 @@ if(NOT PythonLibsNew_FIND_VERSION) set(PythonLibsNew_FIND_VERSION "3.6") endif() +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) + if(NOT _pybind11_cmp0148) + message( + AUTHOR_WARNING + "Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs " + "modules are removed. Run \"cmake --help-policy CMP0148\" for policy " + "details. Use the cmake_policy command to set the policy and suppress " + "this warning, or preferably upgrade to using FindPython, either by " + "calling it explicitly before pybind11, or by setting " + "PYBIND11_FINDPYTHON ON before pybind11.") + endif() + cmake_policy(SET CMP0148 OLD) + unset(_pybind11_cmp0148) +endif() + find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} ${_pythonlibs_required} ${_pythonlibs_quiet}) From 5891867ee4ad1d671e0f54c5fb30f62d1322d8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Papp?= <108480845+lpapp-foundry@users.noreply.github.com> Date: Fri, 15 Sep 2023 23:05:43 +0100 Subject: [PATCH 67/78] fix(cmake): support DEBUG_POSTFIX correctly (#4761) * cmake: split extension Into suffix and debug postfix. Pybind11 is currently treating both as suffix, which is problematic when something else defines the DEBUG_POSTFIX because they will be concatenated. pybind11_extension sets SUFFIX to _d.something and if DEBUG_POSTFIX is set to _d. _d + _d.something = _d_d.something The issue has been reported at: https://github.com/pybind/pybind11/issues/4699 * style: pre-commit fixes * fix(cmake): support postfix for old FindPythonInterp mode too Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- tools/FindPythonLibsNew.cmake | 9 ++++++++- tools/pybind11NewTools.cmake | 34 ++++++++++++++++++++++++---------- tools/pybind11Tools.cmake | 8 ++++++-- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index 992f9f52..8275b9d5 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -188,13 +188,20 @@ _pybind11_get_if_undef(_PYTHON_VALUES 0 _PYTHON_VERSION_LIST) _pybind11_get_if_undef(_PYTHON_VALUES 1 PYTHON_PREFIX) _pybind11_get_if_undef(_PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) _pybind11_get_if_undef(_PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) -_pybind11_get_if_undef(_PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) _pybind11_get_if_undef(_PYTHON_VALUES 5 PYTHON_IS_DEBUG) _pybind11_get_if_undef(_PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) _pybind11_get_if_undef(_PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) _pybind11_get_if_undef(_PYTHON_VALUES 8 PYTHON_LIBDIR) _pybind11_get_if_undef(_PYTHON_VALUES 9 PYTHON_MULTIARCH) +list(GET _PYTHON_VALUES 4 _PYTHON_MODULE_EXT_SUFFIX) +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) +endif() +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) +endif() + # Make sure the Python has the same pointer-size as the chosen compiler # Skip if CMAKE_SIZEOF_VOID_P is not defined # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 3a35a738..8cee2d46 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -95,25 +95,36 @@ endif() # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is # required for PyPy3 (as of 7.3.1) -if(NOT DEFINED PYTHON_MODULE_EXTENSION) +if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) execute_process( COMMAND "${${_Python}_EXECUTABLE}" "-c" "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" - OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION - ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR + OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX + ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR OUTPUT_STRIP_TRAILING_WHITESPACE) - if(_PYTHON_MODULE_EXTENSION STREQUAL "") + if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "") message( FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'" - "package is not installed. Full error message:\n${_PYTHON_MODULE_EXTENSION_ERR}") + "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}" + ) endif() # This needs to be available for the pybind11_extension function - set(PYTHON_MODULE_EXTENSION - "${_PYTHON_MODULE_EXTENSION}" - CACHE INTERNAL "") + if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + CACHE INTERNAL "") + endif() + + if(NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + CACHE INTERNAL "") + endif() endif() # Python debug libraries expose slightly different objects before 3.8 @@ -253,6 +264,9 @@ endfunction() function(pybind11_extension name) # The extension is precomputed - set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${PYTHON_MODULE_EXTENSION}") - + set_target_properties( + ${name} + PROPERTIES PREFIX "" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 67e710bd..1b1774d0 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -65,6 +65,7 @@ _pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS) _pybind11_promote_to_cache(PYTHON_LIBRARIES) _pybind11_promote_to_cache(PYTHON_MODULE_PREFIX) _pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION) +_pybind11_promote_to_cache(PYTHON_MODULE_DEBUG_POSTFIX) _pybind11_promote_to_cache(PYTHON_VERSION_MAJOR) _pybind11_promote_to_cache(PYTHON_VERSION_MINOR) _pybind11_promote_to_cache(PYTHON_VERSION) @@ -148,8 +149,11 @@ endif() function(pybind11_extension name) # The prefix and extension are provided by FindPythonLibsNew.cmake - set_target_properties(${name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") + set_target_properties( + ${name} + PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() # Build a Python extension module: From 7e5edbc947168fa73e25c5b6e58d656f5e2e9863 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Mon, 25 Sep 2023 10:38:21 -0500 Subject: [PATCH 68/78] Avoid copy in iteration by using const auto & (#4861) This change is fixing a Coverity AUTO_CAUSES_COPY issues. --- include/pybind11/pybind11.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 25ca508a..4c0069bf 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1142,7 +1142,7 @@ class cpp_function : public function { } msg += "kwargs: "; bool first = true; - for (auto kwarg : kwargs) { + for (const auto &kwarg : kwargs) { if (first) { first = false; } else { From f468b0707ea8eb1b0f91d7a77bdaec6d1c385586 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 27 Sep 2023 09:55:49 -0700 Subject: [PATCH 69/78] Add 2 missing `throw error_already_set();` (#4863) Fixes oversights in PR #4570. --- include/pybind11/detail/internals.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index e2c94ea9..ab399016 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -451,6 +451,7 @@ inline object get_python_state_dict() { #endif if (!state_dict) { raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + throw error_already_set(); } return state_dict; } @@ -463,6 +464,7 @@ inline internals **get_internals_pp_from_capsule(handle obj) { void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); if (raw_ptr == nullptr) { raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + throw error_already_set(); } return static_cast(raw_ptr); } From 0a756c0bb2f7e7ac6cf4e2c4de26211409fadbe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= <8431159+mtsokol@users.noreply.github.com> Date: Wed, 27 Sep 2023 19:22:04 +0200 Subject: [PATCH 70/78] MAINT: Include `numpy._core` imports (#4857) * MAINT: Include numpy._core imports * style: pre-commit fixes * Apply review comments * style: pre-commit fixes * Add no-inline attribute * Select submodule name based on numpy version * style: pre-commit fixes * Update pre-commit check * Add error_already_set and simplify if statement * Update .pre-commit-config.yaml Co-authored-by: Ralf W. Grosse-Kunstleve --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve --- .pre-commit-config.yaml | 2 +- include/pybind11/numpy.h | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1fb4ae1..c511ab91 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -142,7 +142,7 @@ repos: - id: disallow-caps name: Disallow improper capitalization language: pygrep - entry: PyBind|Numpy|Cmake|CCache|PyTest + entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ # PyLint has native support - not always usable, but works for us diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 02f65d74..23c38660 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -120,6 +120,20 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { + module_ numpy = module_::import("numpy"); + str version_string = numpy.attr("__version__"); + + module_ numpy_lib = module_::import("numpy.lib"); + object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); + int major_version = numpy_version.attr("major").cast(); + + /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially + became a private module. */ + std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core"; + return module_::import((numpy_core_path + "." + submodule_name).c_str()); +} + template struct same_size { template @@ -263,9 +277,13 @@ struct npy_api { }; static npy_api lookup() { - module_ m = module_::import("numpy.core.multiarray"); + module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); + if (api_ptr == nullptr) { + raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + throw error_already_set(); + } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); @@ -626,11 +644,8 @@ class dtype : public object { private: static object _dtype_from_pep3118() { - static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118") - .cast() - .release() - .ptr(); + module_ m = detail::import_numpy_core_submodule("_internal"); + static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); return reinterpret_borrow(obj); } From dd64df73c31fd333018fec34fa7e35e9df365bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Sok=C3=B3=C5=82?= <8431159+mtsokol@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:12:58 +0200 Subject: [PATCH 71/78] MAINT: Remove np.int_ (#4867) --- tests/test_numpy_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 12e7d17d..0697daf3 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -298,7 +298,7 @@ def test_constructors(): results = m.converting_constructors([1, 2, 3]) for a in results.values(): np.testing.assert_array_equal(a, [1, 2, 3]) - assert results["array"].dtype == np.int_ + assert results["array"].dtype == np.dtype(int) assert results["array_t"].dtype == np.int32 assert results["array_t"].dtype == np.float64 From 2b2e4ca4a3fe162bf9c4c95d1621dadab071df86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:13:44 -0400 Subject: [PATCH 72/78] chore(deps): update pre-commit hooks (#4868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update pre-commit hooks updates: - [github.com/psf/black-pre-commit-mirror: 23.7.0 → 23.9.1](https://github.com/psf/black-pre-commit-mirror/compare/23.7.0...23.9.1) - [github.com/astral-sh/ruff-pre-commit: v0.0.287 → v0.0.292](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.287...v0.0.292) - [github.com/codespell-project/codespell: v2.2.5 → v2.2.6](https://github.com/codespell-project/codespell/compare/v2.2.5...v2.2.6) - [github.com/shellcheck-py/shellcheck-py: v0.9.0.5 → v0.9.0.6](https://github.com/shellcheck-py/shellcheck-py/compare/v0.9.0.5...v0.9.0.6) - [github.com/PyCQA/pylint: v3.0.0a7 → v3.0.0](https://github.com/PyCQA/pylint/compare/v3.0.0a7...v3.0.0) * Update .pre-commit-config.yaml * style: pre-commit fixes * Update .pre-commit-config.yaml --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- .github/CONTRIBUTING.md | 2 +- .pre-commit-config.yaml | 10 +++++----- include/pybind11/detail/init.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ad797439..f5a08e2d 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -135,7 +135,7 @@ The valid options are: * Use `-G` and the name of a generator to use something different. `cmake --help` lists the generators available. - On Unix, setting `CMAKE_GENERATER=Ninja` in your environment will give - you automatic mulithreading on all your CMake projects! + you automatic multithreading on all your CMake projects! * Open the `CMakeLists.txt` with QtCreator to generate for that IDE. * You can use `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON` to generate the `.json` file that some tools expect. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c511ab91..e1590204 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,13 +32,13 @@ repos: # Black, the code formatter, natively supports pre-commit - repo: https://github.com/psf/black-pre-commit-mirror - rev: "23.7.0" # Keep in sync with blacken-docs + rev: "23.9.1" # Keep in sync with blacken-docs hooks: - id: black # Ruff, the Python auto-correcting linter written in Rust - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.287 + rev: v0.0.292 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -124,7 +124,7 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.5" + rev: "v2.2.6" hooks: - id: codespell exclude: ".supp$" @@ -132,7 +132,7 @@ repos: # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.9.0.5" + rev: "v0.9.0.6" hooks: - id: shellcheck @@ -147,7 +147,7 @@ repos: # PyLint has native support - not always usable, but works for us - repo: https://github.com/PyCQA/pylint - rev: "v3.0.0a7" + rev: "v3.0.0" hooks: - id: pylint files: ^pybind11 diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index e2117168..4509bd13 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -65,7 +65,7 @@ constexpr bool is_alias(void *) { } // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall -// back to brace aggregate initiailization so that for aggregate initialization can be used with +// back to brace aggregate initialization so that for aggregate initialization can be used with // py::init, e.g. `py::init` to initialize a `struct T { int a; int b; }`. For // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually // works, but will not do the expected thing when `T` has an `initializer_list` constructor). From 6c77208561f23c255bf73e3d76674a6a9496179f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 21:21:22 -0700 Subject: [PATCH 73/78] chore(deps): bump seanmiddleditch/gha-setup-ninja from 3 to 4 (#4875) Bumps [seanmiddleditch/gha-setup-ninja](https://github.com/seanmiddleditch/gha-setup-ninja) from 3 to 4. - [Release notes](https://github.com/seanmiddleditch/gha-setup-ninja/releases) - [Commits](https://github.com/seanmiddleditch/gha-setup-ninja/compare/v3...v4) --- updated-dependencies: - dependency-name: seanmiddleditch/gha-setup-ninja dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fcf0ca5..9d0312b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1052,7 +1052,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.14 - name: Install ninja-build tool - uses: seanmiddleditch/gha-setup-ninja@v3 + uses: seanmiddleditch/gha-setup-ninja@v4 - name: Run pip installs run: | From 0e2c3e5db41b6b2af4038734c84ab855ccaaa5f0 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 11 Oct 2023 21:05:31 -0700 Subject: [PATCH 74/78] Add pybind11/gil_safe_call_once.h (to fix deadlocks in pybind11/numpy.h) (#4877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * LazyInitializeAtLeastOnceDestroyNever v1 * Go back to using `union` as originally suggested by jbms@. The trick (also suggested by jbms@) is to add empty ctor + dtor. * Revert "Go back to using `union` as originally suggested by jbms@. The trick (also suggested by jbms@) is to add empty ctor + dtor." This reverts commit e7b8c4f0fcd72191e88d1c17abf5da08fe3a9c6f. * Remove `#include ` * `include\pybind11/numpy.h(24,10): fatal error C1083: Cannot open include file: 'stdalign.h': No such file or directory` * @tkoeppe wrote: this is a C interop header (and we're not writing C) * Suppress gcc 4.8.5 (CentOS 7) warning. ``` include/pybind11/eigen/../numpy.h:63:53: error: dereferencing type-punned pointer will break strict-aliasing rules [-Werror=strict-aliasing] return *reinterpret_cast(value_storage_); ^ ``` * Replace comments: Document PRECONDITION. Adopt comment suggested by @tkoeppe: https://github.com/pybind/pybind11/pull/4877#discussion_r1350356093 * Adopt suggestion by @tkoeppe: * https://github.com/pybind/pybind11/pull/4877#issuecomment-1752969127 * https://godbolt.org/z/Wa79nKz6e * Add `PYBIND11_CONSTINIT`, but it does not work for the current use cases: ``` g++ -o pybind11/tests/test_numpy_array.os -c -std=c++20 -fPIC -fvisibility=hidden -O0 -g -Wall -Wextra -Wconversion -Wcast-qual -Wdeprecated -Wundef -Wnon-virtual-dtor -Wunused-result -Werror -isystem /usr/include/python3.11 -isystem /usr/include/eigen3 -DPYBIND11_STRICT_ASSERTS_CLASS_HOLDER_VS_TYPE_CASTER_MIX -DPYBIND11_ENABLE_TYPE_CASTER_ODR_GUARD_IF_AVAILABLE -DPYBIND11_TEST_BOOST -Ipybind11/include -I/usr/local/google/home/rwgk/forked/pybind11/include -I/usr/local/google/home/rwgk/clone/pybind11/include /usr/local/google/home/rwgk/forked/pybind11/tests/test_numpy_array.cpp ``` ``` In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_numpy_array.cpp:10: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h: In static member function ‘static pybind11::detail::npy_api& pybind11::detail::npy_api::get()’: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h:258:82: error: ‘constinit’ variable ‘api_init’ does not have a constant initializer 258 | PYBIND11_CONSTINIT static LazyInitializeAtLeastOnceDestroyNever api_init; | ^~~~~~~~ ``` ``` In file included from /usr/local/google/home/rwgk/forked/pybind11/tests/test_numpy_array.cpp:10: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h: In static member function ‘static pybind11::object& pybind11::dtype::_dtype_from_pep3118()’: /usr/local/google/home/rwgk/forked/pybind11/include/pybind11/numpy.h:697:13: error: ‘constinit’ variable ‘imported_obj’ does not have a constant initializer 697 | imported_obj; | ^~~~~~~~~~~~ ``` * Revert "Add `PYBIND11_CONSTINIT`, but it does not work for the current use cases:" This reverts commit f07b28bda9f91fb723aa898a21c81b6dd6857072. * Reapply "Add `PYBIND11_CONSTINIT`, but it does not work for the current use cases:" This reverts commit 36be645758aa82b576d24003808386bec6e55bf9. * Add Default Member Initializer on `value_storage_` as suggested by @tkoeppe: https://github.com/pybind/pybind11/pull/4877#issuecomment-1753201342 This fixes the errors reported under commit f07b28bda9f91fb723aa898a21c81b6dd6857072. * Fix copy-paste-missed-a-change mishap in commit 88cec1152ab5576db19bab95c484672f06f5989a. * Semi-paranoid placement new (based on https://github.com/pybind/pybind11/pull/4877#discussion_r1350573114). * Move PYBIND11_CONSTINIT to detail/common.h * Move code to the right places, rename new class and some variables. * Fix oversight: update tests/extra_python_package/test_files.py * Get the name right first. * Use `std::call_once`, `std::atomic`, following a pattern developed by @tkoeppe * Make the API more self-documenting (and possibly more easily reusable). * google-clang-tidy IWYU fixes * Rewrite comment as suggested by @tkoeppe * Update test_exceptions.cpp and exceptions.rst * Fix oversight in previous commit: add `PYBIND11_CONSTINIT` * Make `get_stored()` non-const for simplicity. As suggested by @tkoeppe: not seeing any reasonable use in which `get_stored` has to be const. * Add comment regarding `KeyboardInterrupt` behavior, based heavily on information provided by @jbms. * Add `assert(PyGILState_Check())` in `gil_scoped_release` ctor (simple & non-simple implementation) as suggested by @EthanSteinberg. * Fix oversight in previous commit (missing include cassert). * Remove use of std::atomic, leaving comments with rationale, why it is not needed. * Rewrite comment re `std:optional` based on deeper reflection (aka 2nd thoughts). * Additional comment with the conclusion of a discussion under PR #4877. * https://github.com/pybind/pybind11/pull/4877#issuecomment-1757363179 * Small comment changes suggested by @tkoeppe. --- CMakeLists.txt | 1 + docs/advanced/exceptions.rst | 9 ++- include/pybind11/detail/common.h | 8 +++ include/pybind11/gil.h | 10 ++- include/pybind11/gil_safe_call_once.h | 91 ++++++++++++++++++++++++ include/pybind11/numpy.h | 19 +++-- tests/extra_python_package/test_files.py | 1 + tests/test_exceptions.cpp | 8 ++- 8 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 include/pybind11/gil_safe_call_once.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 65ad0c22..d2c44dc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ set(PYBIND11_HEADERS include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h + include/pybind11/gil_safe_call_once.h include/pybind11/iostream.h include/pybind11/functional.h include/pybind11/numpy.h diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 616e2d77..e20f42b5 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -141,15 +141,14 @@ standard python RuntimeError: .. code-block:: cpp - // This is a static object, so we must leak the Python reference: - // It is undefined when the destructor will run, possibly only after the - // Python interpreter is finalized already. - static py::handle exc = py::exception(m, "MyCustomError").release(); + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store exc_storage; + exc_storage.call_once_and_store_result( + [&]() { return py::exception(m, "MyCustomError"); }); py::register_exception_translator([](std::exception_ptr p) { try { if (p) std::rethrow_exception(p); } catch (const MyCustomException &e) { - py::set_error(exc, e.what()); + py::set_error(exc_storage.get_stored(), e.what()); } catch (const OtherException &e) { py::set_error(PyExc_RuntimeError, e.what()); } diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index e79f7693..83800e96 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -118,6 +118,14 @@ # endif #endif +#if defined(PYBIND11_CPP20) +# define PYBIND11_CONSTINIT constinit +# define PYBIND11_DTOR_CONSTEXPR constexpr +#else +# define PYBIND11_CONSTINIT +# define PYBIND11_DTOR_CONSTEXPR +#endif + // Compiler version assertions #if defined(__INTEL_COMPILER) # if __INTEL_COMPILER < 1800 diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 570a5581..da22f48d 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -11,6 +11,8 @@ #include "detail/common.h" +#include + #if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # include "detail/internals.h" #endif @@ -137,7 +139,9 @@ class gil_scoped_acquire { class gil_scoped_release { public: + // PRECONDITION: The GIL must be held when this constructor is called. explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + assert(PyGILState_Check()); // `get_internals()` must be called here unconditionally in order to initialize // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an // initialization race could occur as multiple threads try `gil_scoped_acquire`. @@ -201,7 +205,11 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() : state{PyEval_SaveThread()} {} + // PRECONDITION: The GIL must be held when this constructor is called. + gil_scoped_release() { + assert(PyGILState_Check()); + state = PyEval_SaveThread(); + } gil_scoped_release(const gil_scoped_release &) = delete; gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h new file mode 100644 index 00000000..eaf84d16 --- /dev/null +++ b/include/pybind11/gil_safe_call_once.h @@ -0,0 +1,91 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "gil.h" + +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +// Use the `gil_safe_call_once_and_store` class below instead of the naive +// +// static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE! +// +// which has two serious issues: +// +// 1. Py_DECREF() calls potentially after the Python interpreter was finalized already, and +// 2. deadlocks in multi-threaded processes (because of missing lock ordering). +// +// The following alternative avoids both problems: +// +// PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; +// auto &imported_obj = storage // Do NOT make this `static`! +// .call_once_and_store_result([]() { +// return py::module_::import("module_name"); +// }) +// .get_stored(); +// +// The parameter of `call_once_and_store_result()` must be callable. It can make +// CPython API calls, and in particular, it can temporarily release the GIL. +// +// `T` can be any C++ type, it does not have to involve CPython API types. +// +// The behavior with regard to signals, e.g. `SIGINT` (`KeyboardInterrupt`), +// is not ideal. If the main thread is the one to actually run the `Callable`, +// then a `KeyboardInterrupt` will interrupt it if it is running normal Python +// code. The situation is different if a non-main thread runs the +// `Callable`, and then the main thread starts waiting for it to complete: +// a `KeyboardInterrupt` will not interrupt the non-main thread, but it will +// get processed only when it is the main thread's turn again and it is running +// normal Python code. However, this will be unnoticeable for quick call-once +// functions, which is usually the case. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) { + if (!is_initialized_) { // This read is guarded by the GIL. + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + std::call_once(once_flag_, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + ::new (storage_) T(fn()); // fn may release, but will reacquire, the GIL. + is_initialized_ = true; // This write is guarded by the GIL. + }); + // All threads will observe `is_initialized_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + assert(is_initialized_); + PYBIND11_WARNING_PUSH +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 + // Needed for gcc 4.8.5 + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") +#endif + return *reinterpret_cast(storage_); + PYBIND11_WARNING_POP + } + + constexpr gil_safe_call_once_and_store() = default; + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + +private: + alignas(T) char storage_[sizeof(T)] = {}; + std::once_flag once_flag_ = {}; + bool is_initialized_ = false; + // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, + // but the latter does not have the triviality properties of former, + // therefore `std::optional` is not a viable alternative here. +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 23c38660..8551aa26 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -10,7 +10,10 @@ #pragma once #include "pybind11.h" +#include "detail/common.h" #include "complex.h" +#include "gil_safe_call_once.h" +#include "pytypes.h" #include #include @@ -206,8 +209,8 @@ struct npy_api { }; static npy_api &get() { - static npy_api api = lookup(); - return api; + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage.call_once_and_store_result(lookup).get_stored(); } bool PyArray_Check_(PyObject *obj) const { @@ -643,10 +646,14 @@ class dtype : public object { char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; } private: - static object _dtype_from_pep3118() { - module_ m = detail::import_numpy_core_submodule("_internal"); - static PyObject *obj = m.attr("_dtype_from_pep3118").cast().release().ptr(); - return reinterpret_borrow(obj); + static object &_dtype_from_pep3118() { + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([]() { + return detail::import_numpy_core_submodule("_internal") + .attr("_dtype_from_pep3118"); + }) + .get_stored(); } dtype strip_padding(ssize_t itemsize) { diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index e3d881c0..344e70d5 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -35,6 +35,7 @@ "include/pybind11/eval.h", "include/pybind11/functional.h", "include/pybind11/gil.h", + "include/pybind11/gil_safe_call_once.h", "include/pybind11/iostream.h", "include/pybind11/numpy.h", "include/pybind11/operators.h", diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index f6d9c215..d3af7ab7 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -6,6 +6,8 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#include + #include "test_exceptions.h" #include "local_bindings.h" @@ -114,7 +116,9 @@ TEST_SUBMODULE(exceptions, m) { []() { throw std::runtime_error("This exception was intentionally thrown."); }); // PLEASE KEEP IN SYNC with docs/advanced/exceptions.rst - static py::handle ex = py::exception(m, "MyException").release(); + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store ex_storage; + ex_storage.call_once_and_store_result( + [&]() { return py::exception(m, "MyException"); }); py::register_exception_translator([](std::exception_ptr p) { try { if (p) { @@ -122,7 +126,7 @@ TEST_SUBMODULE(exceptions, m) { } } catch (const MyException &e) { // Set MyException as the active python error - py::set_error(ex, e.what()); + py::set_error(ex_storage.get_stored(), e.what()); } }); From 0cbd92bab8c940f8dbb9327aa6a8bdfa8215b589 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Mon, 16 Oct 2023 16:42:30 +0200 Subject: [PATCH 75/78] Update pytest to version 7.2.0 (which removes their dependency on py) (#4880) * Update pytest (which removes their dependency on py) The py library through 1.11.0 for Python allows remote attackers to conduct a ReDoS (Regular expression Denial of Service) attack via a Subversion repository with crafted info data, because the InfoSvnCommand argument is mishandled. The particular codepath in question is the regular expression at py._path.svnurl.InfoSvnCommand.lspattern and is only relevant when dealing with subversion (svn) projects. Notably the codepath is not used in the popular pytest project. The developers of the pytest package have released version 7.2.0 which removes their dependency on py. Users of pytest seeing alerts relating to this advisory may update to version 7.2.0 of pytest to resolve this issue. See https://github.com/pytest-dev/py/issues/287#issuecomment-1290407715 for additional context. * Added conditions so that we keep using 7.0.0 on python 3.6 --- tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 4ba10111..09063e76 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,7 +3,8 @@ numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux" numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6" numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10" numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" and python_version<"3.11" -pytest==7.0.0 +pytest==7.0.0; platform_python_implementation!="PyPy" and python_version=="3.6" +pytest==7.2.0; platform_python_implementation!="PyPy" and python_version>="3.7" pytest-timeout scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" From 74439a64a22850f16323f1e1e09ace2a909b0b61 Mon Sep 17 00:00:00 2001 From: Sergei Izmailov Date: Wed, 18 Oct 2023 04:04:46 +0900 Subject: [PATCH 76/78] feature: Use typed iterators in `make_*iterator` (#4876) --- include/pybind11/pybind11.h | 19 +++++++++++++------ tests/test_sequences_and_iterators.py | 9 +++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4c0069bf..a94f6f0f 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -15,6 +15,7 @@ #include "attr.h" #include "gil.h" #include "options.h" +#include "typing.h" #include #include @@ -2447,7 +2448,7 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, @@ -2465,7 +2466,7 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, @@ -2483,7 +2484,7 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, @@ -2498,8 +2499,10 @@ iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_iterator(Type &value, Extra &&...extra) { return make_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2508,8 +2511,10 @@ iterator make_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_key_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_key_iterator(Type &value, Extra &&...extra) { return make_key_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2518,8 +2523,10 @@ iterator make_key_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_value_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_value_iterator(Type &value, Extra &&...extra) { return make_value_iterator( std::begin(value), std::end(value), std::forward(extra)...); } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index acbe9d89..c13f03dd 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -58,6 +58,15 @@ def test_generalized_iterators_simple(): assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_values()) == [2, 4, 5] +def test_iterator_doc_annotations(): + assert m.IntPairs.nonref.__doc__.endswith("-> Iterator[tuple[int, int]]\n") + assert m.IntPairs.nonref_keys.__doc__.endswith("-> Iterator[int]\n") + assert m.IntPairs.nonref_values.__doc__.endswith("-> Iterator[int]\n") + assert m.IntPairs.simple_iterator.__doc__.endswith("-> Iterator[tuple[int, int]]\n") + assert m.IntPairs.simple_keys.__doc__.endswith("-> Iterator[int]\n") + assert m.IntPairs.simple_values.__doc__.endswith("-> Iterator[int]\n") + + def test_iterator_referencing(): """Test that iterators reference rather than copy their referents.""" vec = m.VectorNonCopyableInt() From 7969049de4c11f1d4955ababd97f340d82d2bf5a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 19 Oct 2023 23:12:37 -0700 Subject: [PATCH 77/78] Comment out failing job, with link to #4889 (#4890) --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d0312b1..e1946da9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,9 +195,10 @@ jobs: matrix: include: # TODO: Fails on 3.10, investigate - - python-version: "3.9" - python-debug: true - valgrind: true + # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4889 + # - python-version: "3.9" + # python-debug: true + # valgrind: true - python-version: "3.11" python-debug: false From 2fa74235a5941dfd54f57ca55966e05230a8a365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Stenberg?= Date: Thu, 19 Oct 2023 16:44:31 +0200 Subject: [PATCH 78/78] Temporary fix for the return value cast problem. pybind11::cast_error: return_value_policy = copy, but the object is non-copyable! May be the same/related issue as described here: pybind#1241 Adapted from: https://github.com/kefeimo/pybind11/commit/338d615e12ce41ee021724551841de3cbe0bc1df --- include/pybind11/detail/type_caster_base.h | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index fc5f1c88..fb5405f7 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -546,19 +546,11 @@ class type_caster_generic { case return_value_policy::copy: if (copy_constructor) { valueptr = copy_constructor(src); + wrapper->owned = true; } else { -#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - std::string type_name(tinfo->cpptype->name()); - detail::clean_type_id(type_name); - throw cast_error("return_value_policy = copy, but type " + type_name - + " is non-copyable!"); -#else - throw cast_error("return_value_policy = copy, but type is " - "non-copyable! (#define PYBIND11_DETAILED_ERROR_MESSAGES or " - "compile in debug mode for details)"); -#endif + valueptr = src; + wrapper->owned = false; } - wrapper->owned = true; break; case return_value_policy::move: