Merge branch 'PyO3:main' into weakref

.github/workflows/build.yml

@@ -24,6 +24,7 @@ jobs:
   build:
     continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) || inputs.rust == 'beta' || inputs.rust == 'nightly' }}
     runs-on: ${{ inputs.os }}
+    if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }}
     steps:
       - uses: actions/checkout@v4
 
@@ -65,6 +66,7 @@ jobs:
         run: nox -s docs
 
       - name: Build (no features)
+        if: ${{ !startsWith(inputs.python-version, 'graalpy') }}
         run: cargo build --lib --tests --no-default-features
 
       # --no-default-features when used with `cargo build/test -p` doesn't seem to work!
@@ -74,7 +76,7 @@ jobs:
           cargo build --no-default-features
 
       # Run tests (except on PyPy, because no embedding API).
-      - if: ${{ !startsWith(inputs.python-version, 'pypy') }}
+      - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }}
         name: Test (no features)
         run: cargo test --no-default-features --lib --tests
 
@@ -85,24 +87,25 @@ jobs:
           cargo test --no-default-features
 
       - name: Build (all additive features)
+        if: ${{ !startsWith(inputs.python-version, 'graalpy') }}
         run: cargo build --lib --tests --no-default-features --features "full ${{ inputs.extra-features }}"
 
       - if: ${{ startsWith(inputs.python-version, 'pypy') }}
         name: Build PyPy (abi3-py37)
         run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}"
 
       # Run tests (except on PyPy, because no embedding API).
-      - if: ${{ !startsWith(inputs.python-version, 'pypy') }}
+      - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }}
         name: Test
         run: cargo test --no-default-features --features "full ${{ inputs.extra-features }}"
 
       # Run tests again, but in abi3 mode
-      - if: ${{ !startsWith(inputs.python-version, 'pypy') }}
+      - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }}
         name: Test (abi3)
         run: cargo test --no-default-features --features "abi3 full ${{ inputs.extra-features }}"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
-      - if: ${{ (!startsWith(inputs.python-version, 'pypy')) && (inputs.python-version != '3.7') }}
+      - if: ${{ (!startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy')) && (inputs.python-version != '3.7') }}
         name: Test (abi3-py37)
         run: cargo test --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}"
 
@@ -120,7 +123,7 @@ jobs:
 
       - uses: dorny/paths-filter@v3
         # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here
-        if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }}
+        if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') }}
         id: ffi-changes
         with:
           base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }}
@@ -135,7 +138,7 @@ jobs:
       - name: Run pyo3-ffi-check
         # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor
         # is pypy 3.9 on windows
-        if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }}
+        if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }}
         run: nox -s ffi-check
 
     env:

.github/workflows/cache-cleanup.yml

@@ -1,6 +1,6 @@
 name: CI Cache Cleanup
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
 
@@ -22,7 +22,7 @@ jobs:
           echo "Deleting caches..."
           for cacheKey in $cacheKeysForPR
           do
-              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+              gh actions-cache delete -R $REPO -B $BRANCH --confirm -- $cacheKey
           done
           echo "Done"
         env:

.github/workflows/ci.yml

@@ -74,9 +74,9 @@ jobs:
         rust: [stable]
         platform: [
           {
-            os: "macos-latest",
-            python-architecture: "x64",
-            rust-target: "x86_64-apple-darwin",
+            os: "macos-14",  # first available arm macos runner
+            python-architecture: "arm64",
+            rust-target: "aarch64-apple-darwin",
           },
           {
             os: "ubuntu-latest",
@@ -119,7 +119,7 @@ jobs:
                 rust-target: "x86_64-unknown-linux-gnu",
               }
     name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }}
-    continue-on-error: ${{ matrix.platform.rust != 'stable' }}
+    continue-on-error: ${{ matrix.rust != 'stable' }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@master
@@ -161,7 +161,12 @@ jobs:
         platform:
           [
             {
-              os: "macos-latest",
+              os: "macos-14",  # first available arm macos runner
+              python-architecture: "arm64",
+              rust-target: "aarch64-apple-darwin",
+            },
+            {
+              os: "macos-13",  # last available x86_64 macos runner
               python-architecture: "x64",
               rust-target: "x86_64-apple-darwin",
             },
@@ -223,11 +228,17 @@ jobs:
           "pypy3.8",
           "pypy3.9",
           "pypy3.10",
+          "graalpy24.0",
         ]
         platform:
           [
+            # for the full matrix, use x86_64 macos runners because not all Python versions
+            # PyO3 supports are available for arm on GitHub Actions. (Availability starts
+            # around Python 3.10, can switch the full matrix to arm once earlier versions
+            # are dropped.)
+            # NB: if this switches to arm, switch the arm job below in the `include` to x86_64
             {
-              os: "macos-latest",
+              os: "macos-13",
               python-architecture: "x64",
               rust-target: "x86_64-apple-darwin",
             },
@@ -287,6 +298,18 @@ jobs:
               }
             extra-features: "multiple-pymethods"
 
+          # test arm macos runner with the latest Python version
+          # NB: if the full matrix switchess to arm, switch to x86_64 here
+          - rust: stable
+            python-version: "3.12"
+            platform:
+              {
+                os: "macos-14",
+                python-architecture: "arm64",
+                rust-target: "aarch64-apple-darwin",
+              }
+            extra-features: "multiple-pymethods"
+
   valgrind:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
     needs: [fmt]
@@ -343,13 +366,13 @@ jobs:
 
   coverage:
     needs: [fmt]
-    name: coverage-${{ matrix.os }}
+    name: coverage ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["windows", "macos", "ubuntu"]
-    runs-on: ${{ matrix.os }}-latest
+        os: ["windows-latest", "macos-14", "ubuntu-latest"]  # first available arm macos runner
+    runs-on: ${{ matrix.os }}
     steps:
-      - if: ${{ github.event_name == 'pull_request' && matrix.os != 'ubuntu' }}
+      - if: ${{ github.event_name == 'pull_request' && matrix.os != 'ubuntu-latest' }}
         id: should-skip
         shell: bash
         run: echo 'skip=true' >> $GITHUB_OUTPUT
@@ -526,6 +549,7 @@ jobs:
           workspaces:
             examples/maturin-starter
           save-if: ${{ github.event_name != 'merge_group' }}
+          key: ${{ matrix.target }}
       - name: Setup cross-compiler
         if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }}
         run: sudo apt-get install -y mingw-w64 llvm

.netlify/build.sh

@@ -48,6 +48,15 @@ done
 # Add latest redirect
 echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects
 
+# some backwards compatbiility redirects
+echo "/latest/building_and_distribution/* /latest/building-and-distribution/:splat 302" >> netlify_build/_redirects
+echo "/latest/building-and-distribution/multiple_python_versions/* /latest/building-and-distribution/multiple-python-versions:splat 302" >> netlify_build/_redirects
+echo "/latest/function/error_handling/* /latest/function/error-handling/:splat 302" >> netlify_build/_redirects
+echo "/latest/getting_started/* /latest/getting-started/:splat 302" >> netlify_build/_redirects
+echo "/latest/python_from_rust/* /latest/python-from-rust/:splat 302" >> netlify_build/_redirects
+echo "/latest/python_typing_hints/* /latest/python-typing-hints/:splat 302" >> netlify_build/_redirects
+echo "/latest/trait_bounds/* /latest/trait-bounds/:splat 302" >> netlify_build/_redirects
+
 ## Add landing page redirect
 if [ "${CONTEXT}" == "deploy-preview" ]; then
     echo "/ /main/" >> netlify_build/_redirects

Architecture.md

@@ -48,7 +48,7 @@ In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[c
 `#[cfg(Py_3_7)]`, and `#[cfg(PyPy)]`.
 `Py_LIMITED_API` corresponds to `#define Py_LIMITED_API` macro in Python/C API.
 With `Py_LIMITED_API`, we can build a Python-version-agnostic binary called an
-[abi3 wheel](https://pyo3.rs/latest/building_and_distribution.html#py_limited_apiabi3).
+[abi3 wheel](https://pyo3.rs/latest/building-and-distribution.html#py_limited_apiabi3).
 `Py_3_7` means that the API is available from Python >= 3.7.
 There are also `Py_3_8`, `Py_3_9`, and so on.
 `PyPy` means that the API definition is for PyPy.
@@ -59,45 +59,24 @@ Those flags are set in [`build.rs`](#6-buildrs-and-pyo3-build-config).
 [`src/types`] contains bindings to [built-in types](https://docs.python.org/3/library/stdtypes.html)
 of Python, such as `dict` and `list`.
 For historical reasons, Python's `object` is called `PyAny` in PyO3 and located in [`src/types/any.rs`].
+
 Currently, `PyAny` is a straightforward wrapper of `ffi::PyObject`, defined as:
 
 ```rust
 #[repr(transparent)]
 pub struct PyAny(UnsafeCell<ffi::PyObject>);
 ```
 
-All built-in types are defined as a C struct.
-For example, `dict` is defined as:
-
-```c
-typedef struct {
-    /* Base object */
-    PyObject ob_base;
-    /* Number of items in the dictionary */
-    Py_ssize_t ma_used;
-    /* Dictionary version */
-    uint64_t ma_version_tag;
-    PyDictKeysObject *ma_keys;
-    PyObject **ma_values;
-} PyDictObject;
-```
-
-However, we cannot access such a specific data structure with `#[cfg(Py_LIMITED_API)]` set.
-Thus, all builtin objects are implemented as opaque types by wrapping `PyAny`, e.g.,:
+Concrete Python objects are implemented by wrapping `PyAny`, e.g.,:
 
 ```rust
 #[repr(transparent)]
 pub struct PyDict(PyAny);
 ```
 
-Note that `PyAny` is not a pointer, and it is usually used as a pointer to the object in the
-Python heap, as `&PyAny`.
-This design choice can be changed
-(see the discussion in [#1056](https://github.com/PyO3/pyo3/issues/1056)).
+These types are not intended to be accessed directly, and instead are used through the `Py<T>` and `Bound<T>` smart pointers.
 
-Since we need lots of boilerplate for implementing common traits for these types
-(e.g., `AsPyPointer`, `AsRef<PyAny>`, and `Debug`), we have some macros in
-[`src/types/mod.rs`].
+We have some macros in [`src/types/mod.rs`] which make it easier to implement APIs for concrete Python types.
 
 ## 3. `PyClass` and related functionalities