diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb972f9..e5c53b3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,10 +1,6 @@ name: CI on: pull_request: - push: - branches: - - main - - 3.* workflow_call: inputs: build-number: @@ -103,11 +99,14 @@ jobs: - uses: actions/checkout@v4.1.7 - name: Set up Python - uses: actions/setup-python@v5.3.0 + uses: actions/setup-python@v5.4.0 with: # Appending -dev ensures that we can always build the dev release. # It's a no-op for versions that have been published. python-version: ${{ needs.config.outputs.PYTHON_VER }}-dev + # Ensure that we *always* use the latest build, not a cached version. + # It's an edge case, but when a new alpha is released, we need to use it ASAP. + check-latest: true - name: Build ${{ matrix.target }} run: | @@ -115,7 +114,7 @@ jobs: make ${{ matrix.target }} BUILD_NUMBER=${{ needs.config.outputs.BUILD_NUMBER }} - name: Upload build artefacts - uses: actions/upload-artifact@v4.4.3 + uses: actions/upload-artifact@v4.6.1 with: name: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz path: dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index a9f69ed..5d2ce75 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python environment - uses: actions/setup-python@v5.3.0 + uses: actions/setup-python@v5.4.0 with: python-version: "3.X" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bfaeba2..0f9beb0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -40,14 +40,14 @@ jobs: needs: [ config, ci ] steps: - name: Get build artifacts - uses: actions/download-artifact@v4.1.8 + uses: actions/download-artifact@v4.1.9 with: pattern: Python-* path: dist merge-multiple: true - name: Create Release - uses: ncipollo/release-action@v1.14.0 + uses: ncipollo/release-action@v1.16.0 with: name: ${{ needs.ci.outputs.PYTHON_VER }}-${{ needs.config.outputs.BUILD_NUMBER }} tag: ${{ needs.ci.outputs.PYTHON_VER }}-${{ needs.config.outputs.BUILD_NUMBER }} diff --git a/Makefile b/Makefile index 52f5135..955207d 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,9 @@ PYTHON_VER=$(basename $(PYTHON_VERSION)) # The binary releases of dependencies, published at: # https://github.com/beeware/cpython-apple-source-deps/releases BZIP2_VERSION=1.0.8-1 -LIBFFI_VERSION=3.4.6-1 -OPENSSL_VERSION=3.0.15-1 -XZ_VERSION=5.6.2-1 +LIBFFI_VERSION=3.4.7-1 +OPENSSL_VERSION=3.0.16-1 +XZ_VERSION=5.6.4-1 # Supported OS OS_LIST=macOS iOS tvOS watchOS @@ -128,10 +128,10 @@ ARCH-$(target)=$$(subst .,,$$(suffix $(target))) ifneq ($(os),macOS) ifeq ($$(findstring simulator,$$(SDK-$(target))),) TARGET_TRIPLE-$(target)=$$(ARCH-$(target))-apple-$$(OS_LOWER-$(target))$$(VERSION_MIN-$(os)) -IS_SIMULATOR-$(target)="False" +IS_SIMULATOR-$(target)=False else TARGET_TRIPLE-$(target)=$$(ARCH-$(target))-apple-$$(OS_LOWER-$(target))$$(VERSION_MIN-$(os))-simulator -IS_SIMULATOR-$(target)="True" +IS_SIMULATOR-$(target)=True endif endif @@ -240,6 +240,9 @@ PYTHON_LIB-$(target)=$$(PYTHON_FRAMEWORK-$(target))/Python PYTHON_BIN-$(target)=$$(PYTHON_INSTALL-$(target))/bin PYTHON_INCLUDE-$(target)=$$(PYTHON_FRAMEWORK-$(target))/Headers PYTHON_STDLIB-$(target)=$$(PYTHON_INSTALL-$(target))/lib/python$(PYTHON_VER) +PYTHON_PLATFORM_CONFIG-$(target)=$$(PYTHON_INSTALL-$(target))/platform-config/$$(ARCH-$(target))-$$(SDK-$(target)) +PYTHON_PLATFORM_SITECUSTOMIZE-$(target)=$$(PYTHON_PLATFORM_CONFIG-$(target))/sitecustomize.py + $$(PYTHON_SRCDIR-$(target))/configure: \ downloads/Python-$(PYTHON_VERSION).tar.gz \ @@ -292,23 +295,35 @@ $$(PYTHON_LIB-$(target)): $$(PYTHON_SRCDIR-$(target))/python.exe # Remove any .orig files produced by the compliance patching process find $$(PYTHON_INSTALL-$(target)) -name "*.orig" -exec rm {} \; -endif - -PYTHON_SITECUSTOMIZE-$(target)=$(PROJECT_DIR)/support/$(PYTHON_VER)/$(os)/platform-site/$(target)/sitecustomize.py -$$(PYTHON_SITECUSTOMIZE-$(target)): - @echo ">>> Create cross-platform sitecustomize.py for $(target)" - mkdir -p $$(dir $$(PYTHON_SITECUSTOMIZE-$(target))) - cat $(PROJECT_DIR)/patch/Python/sitecustomize.$(os).py \ +$$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)): + @echo ">>> Create cross-plaform config for $(target)" + mkdir -p $$(PYTHON_PLATFORM_CONFIG-$(target)) + # Create the cross-platform site definition + echo "import _cross_$$(ARCH-$(target))_$$(SDK-$(target)); import _cross_venv;" \ + > $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_venv.pth + cp $(PROJECT_DIR)/patch/Python/make_cross_venv.py \ + $$(PYTHON_PLATFORM_CONFIG-$(target))/make_cross_venv.py + cp $(PROJECT_DIR)/patch/Python/_cross_venv.py \ + $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_venv.py + cp $$(PYTHON_STDLIB-$(target))/_sysconfig* \ + $$(PYTHON_PLATFORM_CONFIG-$(target)) + cat $(PROJECT_DIR)/patch/Python/_cross_target.py.tmpl \ | sed -e "s/{{os}}/$(os)/g" \ + | sed -e "s/{{platform}}/$$(OS_LOWER-$(target))/g" \ | sed -e "s/{{arch}}/$$(ARCH-$(target))/g" \ + | sed -e "s/{{sdk}}/$$(SDK-$(target))/g" \ | sed -e "s/{{version_min}}/$$(VERSION_MIN-$(os))/g" \ | sed -e "s/{{is_simulator}}/$$(IS_SIMULATOR-$(target))/g" \ - | sed -e "s/{{multiarch}}/$$(ARCH-$(target))-$$(SDK-$(target))/g" \ - | sed -e "s/{{tag}}/$$(OS_LOWER-$(target))-$$(VERSION_MIN-$(os))-$$(ARCH-$(target))-$$(SDK-$(target))/g" \ - > $$(PYTHON_SITECUSTOMIZE-$(target)) + > $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_$$(ARCH-$(target))_$$(SDK-$(target)).py + cat $(PROJECT_DIR)/patch/Python/sitecustomize.py.tmpl \ + | sed -e "s/{{arch}}/$$(ARCH-$(target))/g" \ + | sed -e "s/{{sdk}}/$$(SDK-$(target))/g" \ + > $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)) + +endif -$(target): $$(PYTHON_SITECUSTOMIZE-$(target)) $$(PYTHON_LIB-$(target)) +$(target): $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)) $$(PYTHON_LIB-$(target)) ########################################################################### # Target: Debug @@ -335,6 +350,8 @@ vars-$(target): @echo "PYTHON_BIN-$(target): $$(PYTHON_BIN-$(target))" @echo "PYTHON_INCLUDE-$(target): $$(PYTHON_INCLUDE-$(target))" @echo "PYTHON_STDLIB-$(target): $$(PYTHON_STDLIB-$(target))" + @echo "PYTHON_PLATFORM_CONFIG-$(target): $$(PYTHON_PLATFORM_CONFIG-$(target))" + @echo "PYTHON_PLATFORM_SITECUSTOMIZE-$(target): $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target))" @echo endef # build-target @@ -380,6 +397,7 @@ PYTHON_FRAMEWORK-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/Python.framework PYTHON_INSTALL_VERSION-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Versions/$(PYTHON_VER) PYTHON_LIB-$(sdk)=$$(PYTHON_INSTALL_VERSION-$(sdk))/Python PYTHON_INCLUDE-$(sdk)=$$(PYTHON_INSTALL_VERSION-$(sdk))/include/python$(PYTHON_VER) +PYTHON_MODULEMAP-$(sdk)=$$(PYTHON_INCLUDE-$(sdk))/module.modulemap PYTHON_STDLIB-$(sdk)=$$(PYTHON_INSTALL_VERSION-$(sdk))/lib/python$(PYTHON_VER) else @@ -390,11 +408,13 @@ else # The non-macOS frameworks don't use the versioning structure. PYTHON_INSTALL-$(sdk)=$(PROJECT_DIR)/install/$(os)/$(sdk)/python-$(PYTHON_VERSION) +PYTHON_MODULEMAP-$(sdk)=$$(PYTHON_INCLUDE-$(sdk))/module.modulemap PYTHON_FRAMEWORK-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/Python.framework PYTHON_LIB-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Python PYTHON_BIN-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/bin PYTHON_INCLUDE-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Headers PYTHON_STDLIB-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/lib/python$(PYTHON_VER) +PYTHON_PLATFORM_CONFIG-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/platform-config $$(PYTHON_LIB-$(sdk)): $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_LIB-$$(target))) @echo ">>> Build Python fat library for the $(sdk) SDK" @@ -419,6 +439,15 @@ $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h: $$(PYTHON_LIB-$(sdk)) # Copy headers as-is from the first target in the $(sdk) SDK cp -r $$(PYTHON_INCLUDE-$$(firstword $$(SDK_TARGETS-$(sdk)))) $$(PYTHON_INCLUDE-$(sdk)) + # Create the modulemap file + cp -r patch/Python/module.modulemap.prefix $$(PYTHON_MODULEMAP-$(sdk)) + echo "" >> $$(PYTHON_MODULEMAP-$(sdk)) + cd $$(PYTHON_SRCDIR-$$(firstword $$(SDK_TARGETS-$(sdk))))/Include && \ + find cpython -name "*.h" | sort | sed -e 's/^/ exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-$(sdk)) && \ + echo "" >> $$(PYTHON_MODULEMAP-$(sdk)) && \ + find internal -name "*.h" | sort | sed -e 's/^/ exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-$(sdk)) + echo "\n}" >> $$(PYTHON_MODULEMAP-$(sdk)) + # Link the PYTHONHOME version of the headers mkdir -p $$(PYTHON_INSTALL-$(sdk))/include ln -si ../Python.framework/Headers $$(PYTHON_INSTALL-$(sdk))/include/python$(PYTHON_VER) @@ -430,7 +459,7 @@ $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h: $$(PYTHON_LIB-$(sdk)) cp $$(PYTHON_SRCDIR-$$(firstword $$(SDK_TARGETS-$(sdk))))/$(os)/Resources/pyconfig.h $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h -$$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-$(sdk))/Info.plist $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h +$$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-$(sdk))/Info.plist $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target))) @echo ">>> Build Python stdlib for the $(sdk) SDK" mkdir -p $$(PYTHON_STDLIB-$(sdk))/lib-dynload # Copy stdlib from the first target associated with the $(sdk) SDK @@ -445,6 +474,10 @@ $$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK- # Copy the individual _sysconfigdata modules into names that include the architecture $$(foreach target,$$(SDK_TARGETS-$(sdk)),cp $$(PYTHON_STDLIB-$$(target))/_sysconfigdata_* $$(PYTHON_STDLIB-$(sdk))/; ) + # Copy the platform site folders for each architecture + mkdir -p $$(PYTHON_PLATFORM_CONFIG-$(sdk)) + $$(foreach target,$$(SDK_TARGETS-$(sdk)),cp -r $$(PYTHON_PLATFORM_CONFIG-$$(target)) $$(PYTHON_PLATFORM_CONFIG-$(sdk)); ) + # Merge the binary modules from each target in the $(sdk) SDK into a single binary $$(foreach module,$$(wildcard $$(PYTHON_STDLIB-$$(firstword $$(SDK_TARGETS-$(sdk))))/lib-dynload/*),lipo -create -output $$(PYTHON_STDLIB-$(sdk))/lib-dynload/$$(notdir $$(module)) $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_STDLIB-$$(target))/lib-dynload/$$(notdir $$(module))); ) @@ -529,6 +562,15 @@ $$(PYTHON_XCFRAMEWORK-$(os))/Info.plist: \ # Rewrite the framework to make it standalone patch/make-relocatable.sh $$(PYTHON_INSTALL_VERSION-macosx) 2>&1 > /dev/null + # Create the modulemap file + cp -r patch/Python/module.modulemap.prefix $$(PYTHON_MODULEMAP-macosx) + echo "" >> $$(PYTHON_MODULEMAP-macosx) + cd $$(PYTHON_INCLUDE-macosx) && \ + find cpython -name "*.h" | sort | sed -e 's/^/ exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-macosx) && \ + echo "" >> $$(PYTHON_MODULEMAP-macosx) && \ + find internal -name "*.h" | sort | sed -e 's/^/ exclude header "/' | sed 's/$$$$/"/' >> $$(PYTHON_MODULEMAP-macosx) + echo "\n}" >> $$(PYTHON_MODULEMAP-macosx) + # Re-apply the signature on the binaries. codesign -s - --preserve-metadata=identifier,entitlements,flags,runtime -f $$(PYTHON_LIB-macosx) \ 2>&1 | tee $$(PYTHON_INSTALL-macosx)/python-$(os).codesign.log @@ -552,7 +594,7 @@ support/$(PYTHON_VER)/macOS/VERSIONS: dist/Python-$(PYTHON_VER)-macOS-support.$(BUILD_NUMBER).tar.gz: \ $$(PYTHON_XCFRAMEWORK-macOS)/Info.plist \ support/$(PYTHON_VER)/macOS/VERSIONS \ - $$(foreach target,$$(TARGETS-macOS), $$(PYTHON_SITECUSTOMIZE-$$(target))) + $$(foreach target,$$(TARGETS-macOS), $$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target))) @echo ">>> Create final distribution artefact for macOS" mkdir -p dist @@ -575,9 +617,7 @@ $$(PYTHON_XCFRAMEWORK-$(os))/Info.plist: \ $$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/include $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); ) $$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/bin $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); ) $$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/lib $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); ) - - @echo ">>> Create helper links in XCframework for $(os)" - $$(foreach sdk,$$(SDKS-$(os)),ln -si $$(SDK_SLICE-$$(sdk)) $$(PYTHON_XCFRAMEWORK-$(os))/$$(sdk); ) + $$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/platform-config $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); ) ifeq ($(os),iOS) @echo ">>> Clone testbed project for $(os)" @@ -596,7 +636,7 @@ endif dist/Python-$(PYTHON_VER)-$(os)-support.$(BUILD_NUMBER).tar.gz: \ $$(PYTHON_XCFRAMEWORK-$(os))/Info.plist \ - $$(foreach target,$$(TARGETS-$(os)), $$(PYTHON_SITECUSTOMIZE-$$(target))) + $$(foreach target,$$(TARGETS-$(os)), $$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target))) @echo ">>> Create final distribution artefact for $(os)" mkdir -p dist diff --git a/README.rst b/README.rst index 3d9364e..aca2d07 100644 --- a/README.rst +++ b/README.rst @@ -83,15 +83,6 @@ Each support package contains: * ``VERSIONS``, a text file describing the specific versions of code used to build the support package; -* ``platform-site``, a folder that contains site customization scripts that can be used - to make your local Python install look like it is an on-device install for each of the - underlying target architectures supported by the platform. This is needed because when - you run ``pip`` you'll be on a macOS machine with a specific architecture; if ``pip`` - tries to install a binary package, it will install a macOS binary wheel (which won't - work on iOS/tvOS/watchOS). However, if you add the ``platform-site`` folder to your - ``PYTHONPATH`` when invoking pip, the site customization will make your Python install - return ``platform`` and ``sysconfig`` responses consistent with on-device behavior, - which will cause ``pip`` to install platform-appropriate packages. * ``Python.xcframework``, a multi-architecture build of the Python runtime library On iOS/tvOS/watchOS, the ``Python.xcframework`` contains a @@ -105,6 +96,32 @@ needed to build packages. This is required because Xcode uses the ``xcrun`` alias to dynamically generate the name of binaries, but a lot of C tooling expects that ``CC`` will not contain spaces. +Each slice of an iOS/tvOS/watchOS XCframework also contains a +``platform-config`` folder with a subfolder for each supported architecture in +that slice. These subfolders can be used to make a macOS Python environment +behave as if it were on an iOS/tvOS/watchOS device. This works in one of two +ways: + +1. **A sitecustomize.py script**. If the ``platform-config`` subfolder is on + your ``PYTHONPATH`` when a Python interpreter is started, a site + customization will be applied that patches methods in ``sys``, ``sysconfig`` + and ``platform`` that are used to identify the system. + +2. **A make_cross_venv.py script**. If you call ``make_cross_venv.py``, + providing the location of a virtual environment, the script will add some + files to the ``site-packages`` folder of that environment that will + automatically apply the same set of patches as the ``sitecustomize.py`` + script whenever the environment is activated, without any need to modify + ``PYTHONPATH``. If you use ``build`` to create an isolated PEP 517 + environment to build a wheel, these patches will also be applied to the + isolated build environment that is created. + +iOS distributions also contain a copy of the iOS ``testbed`` project - an Xcode +project that can be used to run test suites of Python code. See the `CPython +documentation on testing packages +`__ for +details on how to use this testbed. + For a detailed instructions on using the support package in your own project, see the `usage guide <./USAGE.md>`__ diff --git a/USAGE.md b/USAGE.md index 12bc1dc..096f71b 100644 --- a/USAGE.md +++ b/USAGE.md @@ -20,71 +20,110 @@ what Briefcase is doing). The steps required are documented in the CPython usage guides: * [macOS](https://docs.python.org/3/using/mac.html) -* [iOS](https://docs.python.org/3.14/using/ios.html) +* [iOS](https://docs.python.org/3/using/ios.html#adding-python-to-an-ios-project) For tvOS and watchOS, you should be able to broadly follow the instructions in the iOS guide. +### Using Objective C + +Once you've added the Python XCframework to your project, you'll need to +initialize the Python runtime in your Objective C code (This is step 10 of the +iOS guide linked above). This initialization should generally be done as early +as possible in the application's lifecycle, but definitely needs to be done +before you invoke Python code. + +As a *bare minimum*, you can do the following: + +1. Import the Python C API headers: + ```objc + #include + ``` + +2. Initialize the Python interpreter: + ```objc + NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; + NSString *pythonHome = [NSString stringWithFormat:@"%@/python", resourcePath, nil]; + NSString *pythonPath = [NSString stringWithFormat:@"%@/lib/python3.13", python_home, nil]; + NSString *libDynloadPath = [NSString stringWithFormat:@"%@/lib/python3.13/lib-dynload", python_home, nil]; + NSString *appPath = [NSString stringWithFormat:@"%@/app", resourcePath, nil]; + + setenv("PYTHONHOME", pythonHome, 1); + setenv("PYTHONPATH", [NSString stringWithFormat:@"%@:%@:%@", pythonpath, libDynloadPath, appPath, nil]); + + Py_Initialize(); + + // we now have a Python interpreter ready to be used + ``` + References to a specific Python version should reflect the version of + Python you are using. + +Again - this is the *bare minimum* initialization. In practice, you will likely +need to configure other aspects of the Python interpreter using the +`PyPreConfig` and `PyConfig` mechanisms. Consult the [Python documentation on +interpreter configuration](https://docs.python.org/3/c-api/init_config.html) for +more details on the configuration options that are available. You may find the +[bootstrap mainline code used by +Briefcase](https://github.com/beeware/briefcase-iOS-Xcode-template/blob/main/%7B%7B%20cookiecutter.format%20%7D%7D/%7B%7B%20cookiecutter.class_name%20%7D%7D/main.m) +a helpful point of comparison. + +### Using Swift + +If you want to use Swift instead of Objective C, the bare minimum initialization +code will look something like this: + +1. Import the Python framework: + ```swift + import Python + ``` + +2. Initialize the Python interpreter: + ```swift + guard let pythonHome = Bundle.main.path(forResource: "python", ofType: nil) else { return } + guard let pythonPath = Bundle.main.path(forResource: "python/lib/python3.13", ofType: nil) else { return } + guard let libDynloadPath = Bundle.main.path(forResource: "python/lib/python3.13/lib-dynload", ofType: nil) else { return } + let appPath = Bundle.main.path(forResource: "app", ofType: nil) + + setenv("PYTHONHOME", pythonHome, 1) + setenv("PYTHONPATH", [pythonPath, libDynloadPath, appPath].compactMap { $0 }.joined(separator: ":"), 1) + Py_Initialize() + // we now have a Python interpreter ready to be used + ``` + + Again, references to a specific Python version should reflect the version of + Python you are using; and you will likely need to use `PyPreConfig` and + `PreConfig` APIs. + ## Accessing the Python runtime There are 2 ways to access the Python runtime in your project code. -### Embedded C API. +### Embedded C API You can use the [Python Embedded C -API](https://docs.python.org/3/extending/embedding.html) to instantiate a Python -interpreter. This is the approach taken by Briefcase; you may find the bootstrap -mainline code generated by Briefcase a helpful guide to what is needed to start -an interpreter and run Python code. +API](https://docs.python.org/3/extending/embedding.html) to invoke Python code +and interact with Python objects. This is a raw C API that is accesible to both +Objective C and Swift. ### PythonKit -An alternate approach is to use +If you're using Swift, an alternate approach is to use [PythonKit](https://github.com/pvieito/PythonKit). PythonKit is a package that provides a Swift API to running Python code. -To use PythonKit in your project: - -1. Add PythonKit to your project using the Swift Package manager. See the - PythonKit documentation for details. - -2. Create a file called `module.modulemap` inside - `Python.xcframework/macos-arm64_x86_64/Headers/`, containing the following - code: -``` -module Python { - umbrella header "Python.h" - export * - link "Python" -} -``` +To use PythonKit in your project, add the Python Apple Support package to your +project and instantiate a Python interpreter as described above; then add +PythonKit to your project using the Swift Package manager (see the [PythonKit +documentation](https://github.com/pvieito/PythonKit) for details). -3. In your Swift code, initialize the Python runtime. This should generally be - done as early as possible in the application's lifecycle, but definitely - needs to be done before you invoke Python code: +Once you've done this, you can import PythonKit: ```swift -import Python - -guard let stdLibPath = Bundle.main.path(forResource: "python-stdlib", ofType: nil) else { return } -guard let libDynloadPath = Bundle.main.path(forResource: "python-stdlib/lib-dynload", ofType: nil) else { return } -setenv("PYTHONHOME", stdLibPath, 1) -setenv("PYTHONPATH", "\(stdLibPath):\(libDynloadPath)", 1) -Py_Initialize() -// we now have a Python interpreter ready to be used +import PythonKit ``` - -5. Invoke Python code in your app. For example: +and use the PythonKit Swift API to interact with Python code: ```swift -import PythonKit - let sys = Python.import("sys") print("Python Version: \(sys.version_info.major).\(sys.version_info.minor)") print("Python Encoding: \(sys.getdefaultencoding().upper())") print("Python Path: \(sys.path)") - -_ = Python.import("math") // verifies `lib-dynload` is found and signed successfully ``` - -To integrate 3rd party python code and dependencies, you will need to make sure -`PYTHONPATH` contains their paths; once this has been done, you can run -`Python.import("")`. to import that module from inside swift. diff --git a/patch/Python/Python.patch b/patch/Python/Python.patch index 4b49b3f..820af1d 100644 --- a/patch/Python/Python.patch +++ b/patch/Python/Python.patch @@ -1,25 +1,3 @@ -diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst -index 15794fb5fc5..e3999a60f16 100644 ---- a/Doc/c-api/init_config.rst -+++ b/Doc/c-api/init_config.rst -@@ -1101,6 +1101,17 @@ - - Default: ``1`` in Python config and ``0`` in isolated config. - -+ .. c:member:: int use_system_logger -+ -+ If non-zero, ``stdout`` and ``stderr`` will be redirected to the system -+ log. -+ -+ Only available on macOS 10.12 and later, and on iOS. -+ -+ Default: ``0`` (don't use system log). -+ -+ .. versionadded:: 3.13.2 -+ - .. c:member:: int user_site_directory - - If non-zero, add the user site directory to :data:`sys.path`. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 21c19903464..61baa722359 100644 --- a/Doc/library/importlib.rst @@ -345,20 +323,6 @@ index 9ae0270eaee..2bb14d88dc9 100644 Other Resources =============== -diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h -index 583165bee48..a0f3c418360 100644 ---- a/Include/cpython/initconfig.h -+++ b/Include/cpython/initconfig.h -@@ -173,6 +173,9 @@ - int legacy_windows_stdio; - #endif - wchar_t *check_hash_pycs_mode; -+#ifdef __APPLE__ -+ int use_system_logger; -+#endif - - /* --- Path configuration inputs ------------ */ - int pathconfig_warnings; --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ @@ -1412,11 +1376,11 @@ index 38c499bf722..8de55306053 100644 @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog -+from test.support import is_apple ++from test.support import is_apple_mobile +from unittest.mock import Mock, call + -+if not is_apple: -+ raise unittest.SkipTest("Apple-specific") ++if not is_apple_mobile: ++ raise unittest.SkipTest("iOS-specific") + + +# Test redirection of stdout and stderr to the Apple system log. @@ -2046,7 +2010,7 @@ index 8a436ad123b..4fa7571fe43 100644 command = self.generate_trace_command(script_file, subcommand) stdout, _ = subprocess.Popen(command, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py -index 8c343f37210..f36c1aef84e 100644 +index 8c343f37210..f13b5a1745b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -72,6 +72,7 @@ @@ -2057,16 +2021,7 @@ index 8c343f37210..f36c1aef84e 100644 def run_embedded_interpreter(self, *args, env=None, timeout=None, returncode=0, input=None, cwd=None): -@@ -438,6 +439,8 @@ - CONFIG_COMPAT.update({ - 'legacy_windows_stdio': 0, - }) -+ if support.is_apple: -+ CONFIG_COMPAT['use_system_logger'] = False - - CONFIG_PYTHON = dict(CONFIG_COMPAT, - _config_init=API_PYTHON, -@@ -1421,6 +1424,7 @@ +@@ -1421,6 +1422,7 @@ class SetConfigTests(unittest.TestCase): @@ -11693,50 +11648,6 @@ index e77ca4c2194..724aefabdee 100644 + 18,45,18,25,14,53,14,69,14,49,0,127,14,32,0,127, + 16,28,10,42,8,23,8,18,12,5, }; -diff --git a/Python/initconfig.c b/Python/initconfig.c -index 1e10e6659cb..a3e80eb196e 100644 ---- a/Python/initconfig.c -+++ b/Python/initconfig.c -@@ -644,6 +644,9 @@ - assert(config->check_hash_pycs_mode != NULL); - assert(config->_install_importlib >= 0); - assert(config->pathconfig_warnings >= 0); -+#ifdef __APPLE__ -+ assert(config->use_system_logger >= 0); -+#endif - return 1; - } - #endif -@@ -728,6 +731,9 @@ - #ifdef MS_WINDOWS - config->legacy_windows_stdio = -1; - #endif -+#ifdef __APPLE__ -+ config->use_system_logger = 0; -+#endif - } - - /* Excluded from public struct PyConfig for backporting reasons. */ -@@ -757,6 +763,9 @@ - #ifdef MS_WINDOWS - config->legacy_windows_stdio = 0; - #endif -+#ifdef __APPLE__ -+ config->use_system_logger = 0; -+#endif - } - - -@@ -789,6 +798,9 @@ - #ifdef MS_WINDOWS - config->legacy_windows_stdio = 0; - #endif -+#ifdef __APPLE__ -+ config->use_system_logger = 0; -+#endif - } - - diff --git a/Python/marshal.c b/Python/marshal.c index 67540e08ed9..062e3047fb8 100644 --- a/Python/marshal.c @@ -11768,40 +11679,50 @@ index 67540e08ed9..062e3047fb8 100644 #define TYPE_NULL '0' diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c -index b0c5da56c4f..3e1c8674030 100644 +index b0c5da56c4f..ec53e7484fc 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c -@@ -18,7 +18,9 @@ +@@ -18,7 +18,21 @@ #include // setlocale() #if defined(__APPLE__) -#include +# include ++# include +# include -+# include ++// The os_log unified logging APIs were introduced in macOS 10.12, iOS 10.0, ++// tvOS 10.0, and watchOS 3.0; we enable the use of the system logger ++// automatically on non-macOS platforms. ++# if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE ++# define USE_APPLE_SYSTEM_LOG 1 ++# else ++# define USE_APPLE_SYSTEM_LOG 0 ++# endif ++ ++# if USE_APPLE_SYSTEM_LOG ++# include ++# endif #endif #ifdef HAVE_SIGNAL_H -@@ -58,6 +60,9 @@ +@@ -58,6 +72,9 @@ static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); -+#if defined(__APPLE__) ++#if defined(__APPLE__) && USE_APPLE_SYSTEM_LOG +static PyStatus init_apple_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); -@@ -1126,6 +1131,19 @@ +@@ -1126,6 +1143,17 @@ return status; } -+#if defined(__APPLE__) -+ if (config->use_system_logger) { -+ status = init_apple_streams(tstate); -+ if (_PyStatus_EXCEPTION(status)) { -+ return status; -+ } ++#if defined(__APPLE__) && USE_APPLE_SYSTEM_LOG ++ status = init_apple_streams(tstate); ++ if (_PyStatus_EXCEPTION(status)) { ++ return status; + } +#endif + @@ -11812,11 +11733,11 @@ index b0c5da56c4f..3e1c8674030 100644 status = add_main_module(interp); if (_PyStatus_EXCEPTION(status)) { return status; -@@ -2408,6 +2426,75 @@ +@@ -2408,6 +2436,69 @@ return res; } -+#if defined(__APPLE__) ++#if defined(__APPLE__) && USE_APPLE_SYSTEM_LOG + +static PyObject * +apple_log_write_impl(PyObject *self, PyObject *args) @@ -11827,14 +11748,9 @@ index b0c5da56c4f..3e1c8674030 100644 + return NULL; + } + -+ // Call the underlying Apple logging API. The os_log unified logging APIs -+ // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0; -+ // this call is a no-op on older versions. -+ #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) + // Pass the user-provided text through explicit %s formatting + // to avoid % literals being interpreted as a formatting directive. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); -+ #endif + Py_RETURN_NONE; +} + @@ -11869,7 +11785,6 @@ index b0c5da56c4f..3e1c8674030 100644 + if (result == NULL) { + goto error; + } -+ + goto done; + +error: @@ -11883,7 +11798,7 @@ index b0c5da56c4f..3e1c8674030 100644 + return status; +} + -+#endif // __APPLE__ ++#endif // __APPLE__ && USE_APPLE_SYSTEM_LOG + static void @@ -15787,11 +15702,12 @@ index ac3be3850a9..4bfd669aa87 100644 +build for testing purposes (either x86_64 or ARM64). --- /dev/null +++ b/iOS/testbed/__main__.py -@@ -0,0 +1,407 @@ +@@ -0,0 +1,468 @@ +import argparse +import asyncio +import json +import plistlib ++import re +import shutil +import subprocess +import sys @@ -15803,6 +15719,18 @@ index ac3be3850a9..4bfd669aa87 100644 + +DECODE_ARGS = ("UTF-8", "backslashreplace") + ++# The system log prefixes each line: ++# 2025-01-17 16:14:29.090 Df iOSTestbed[23987:1fd393b4] (Python) ... ++# 2025-01-17 16:14:29.090 E iOSTestbed[23987:1fd393b4] (Python) ... ++ ++LOG_PREFIX_REGEX = re.compile( ++ r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD ++ r"\s+\d+:\d{2}:\d{2}\.\d+" # HH:MM:SS.sss ++ r"\s+\w+" # Df/E ++ r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID ++ r"\s+\(Python\)\s" # Logger name ++) ++ + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). @@ -15860,19 +15788,29 @@ index ac3be3850a9..4bfd669aa87 100644 + +# Return a list of UDIDs associated with booted simulators +async def list_devices(): -+ # List the testing simulators, in JSON format -+ raw_json = await async_check_output( -+ "xcrun", "simctl", "--set", "testing", "list", "-j" -+ ) -+ json_data = json.loads(raw_json) -+ -+ # Filter out the booted iOS simulators -+ return [ -+ simulator["udid"] -+ for runtime, simulators in json_data["devices"].items() -+ for simulator in simulators -+ if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" -+ ] ++ try: ++ # List the testing simulators, in JSON format ++ raw_json = await async_check_output( ++ "xcrun", "simctl", "--set", "testing", "list", "-j" ++ ) ++ json_data = json.loads(raw_json) ++ ++ # Filter out the booted iOS simulators ++ return [ ++ simulator["udid"] ++ for runtime, simulators in json_data["devices"].items() ++ for simulator in simulators ++ if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" ++ ] ++ except subprocess.CalledProcessError as e: ++ # If there's no ~/Library/Developer/XCTestDevices folder (which is the ++ # case on fresh installs, and in some CI environments), `simctl list` ++ # returns error code 1, rather than an empty list. Handle that case, ++ # but raise all other errors. ++ if e.returncode == 1: ++ return [] ++ else: ++ raise + + +async def find_device(initial_devices): @@ -15922,6 +15860,8 @@ index ac3be3850a9..4bfd669aa87 100644 + ) as process: + suppress_dupes = False + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): ++ # Strip the prefix from each log line ++ line = LOG_PREFIX_REGEX.sub("", line) + # The iOS log streamer can sometimes lag; when it does, it outputs + # a warning about messages being dropped... often multiple times. + # Only print the first of these duplicated warnings. @@ -16017,33 +15957,69 @@ index ac3be3850a9..4bfd669aa87 100644 + shutil.copytree(source, target, symlinks=True) + print(" done") + ++ xc_framework_path = target / "Python.xcframework" ++ sim_framework_path = xc_framework_path / "ios-arm64_x86_64-simulator" + if framework is not None: + if framework.suffix == ".xcframework": + print(" Installing XCFramework...", end="", flush=True) -+ xc_framework_path = (target / "Python.xcframework").resolve() + if xc_framework_path.is_dir(): + shutil.rmtree(xc_framework_path) + else: -+ xc_framework_path.unlink() ++ xc_framework_path.unlink(missing_ok=True) + xc_framework_path.symlink_to( + relative_to(framework, xc_framework_path.parent) + ) + print(" done") + else: + print(" Installing simulator framework...", end="", flush=True) -+ sim_framework_path = ( -+ target / "Python.xcframework" / "ios-arm64_x86_64-simulator" -+ ).resolve() + if sim_framework_path.is_dir(): + shutil.rmtree(sim_framework_path) + else: -+ sim_framework_path.unlink() ++ sim_framework_path.unlink(missing_ok=True) + sim_framework_path.symlink_to( + relative_to(framework, sim_framework_path.parent) + ) + print(" done") + else: -+ print(" Using pre-existing iOS framework.") ++ if ( ++ xc_framework_path.is_symlink() ++ and not xc_framework_path.readlink().is_absolute() ++ ): ++ # XCFramework is a relative symlink. Rewrite the symlink relative ++ # to the new location. ++ print(" Rewriting symlink to XCframework...", end="", flush=True) ++ orig_xc_framework_path = ( ++ source ++ / xc_framework_path.readlink() ++ ).resolve() ++ xc_framework_path.unlink() ++ xc_framework_path.symlink_to( ++ orig_xc_framework_path.relative_to( ++ xc_framework_path.parent, walk_up=True ++ ) ++ ) ++ print(" done") ++ elif ( ++ sim_framework_path.is_symlink() ++ and not sim_framework_path.readlink().is_absolute() ++ ): ++ print(" Rewriting symlink to simulator framework...", end="", flush=True) ++ # Simulator framework is a relative symlink. Rewrite the symlink ++ # relative to the new location. ++ orig_sim_framework_path = ( ++ source ++ / "Python.XCframework" ++ / sim_framework_path.readlink() ++ ).resolve() ++ sim_framework_path.unlink() ++ sim_framework_path.symlink_to( ++ orig_sim_framework_path.relative_to( ++ sim_framework_path.parent, walk_up=True ++ ) ++ ) ++ print(" done") ++ else: ++ print(" Using pre-existing iOS framework.") + + for app_src in apps: + print(f" Installing app {app_src.name!r}...", end="", flush=True) @@ -16159,8 +16135,8 @@ index ac3be3850a9..4bfd669aa87 100644 + + if context.subcommand == "clone": + clone_testbed( -+ source=Path(__file__).parent, -+ target=Path(context.location), ++ source=Path(__file__).parent.resolve(), ++ target=Path(context.location).resolve(), + framework=Path(context.framework).resolve() if context.framework else None, + apps=[Path(app) for app in context.apps], + ) @@ -17000,7 +16976,7 @@ index ac3be3850a9..4bfd669aa87 100644 +} --- /dev/null +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m -@@ -0,0 +1,162 @@ +@@ -0,0 +1,160 @@ +#import +#import + @@ -17031,7 +17007,7 @@ index ac3be3850a9..4bfd669aa87 100644 + // Xcode log can't display color. Stdout will report that it is *not* a + // TTY. + setenv("NO_COLOR", "1", true); -+ setenv("PY_COLORS", "0", true); ++ setenv("PYTHON_COLORS", "0", true); + + // Arguments to pass into the test suite runner. + // argv[0] must identify the process; any subsequent arg @@ -17056,8 +17032,6 @@ index ac3be3850a9..4bfd669aa87 100644 + // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. + // See https://docs.python.org/3/library/os.html#python-utf-8-mode. + preconfig.utf8_mode = 1; -+ // Use the system logger for stdout/err -+ config.use_system_logger = 1; + // Don't buffer stdio. We want output to appears in the log immediately + config.buffered_stdio = 0; + // Don't write bytecode; we can't modify the app bundle diff --git a/patch/Python/_cross_target.py.tmpl b/patch/Python/_cross_target.py.tmpl new file mode 100644 index 0000000..5f822e2 --- /dev/null +++ b/patch/Python/_cross_target.py.tmpl @@ -0,0 +1,78 @@ +# A site package that turns a macOS virtual environment +# into an {{arch}} {{sdk}} cross-platform virtual environment +import collections +import platform +import subprocess +import sys +import sysconfig + +########################################################################### +# sys module patches +########################################################################### +sys.cross_compiling = True +sys.platform = "{{platform}}" +sys.implementation._multiarch = "{{arch}}-{{sdk}}" + +########################################################################### +# subprocess module patches +########################################################################### +subprocess._can_fork_exec = True + + +########################################################################### +# platform module patches +########################################################################### + +def cross_system(): + return "{{os}}" + + +def cross_uname(): + return platform.uname_result( + system="{{os}}", + node="build", + release="{{version_min}}", + version="", + machine="{{arch}}", + ) + + +platform.IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def cross_ios_ver(system="", release="", model="", is_simulator=False): + if system == "": + system = "{{os}}" + if release == "": + release = "{{version_min}}" + if model == "": + model = "{{sdk}}" + + return platform.IOSVersionInfo(system, release, model, {{is_simulator}}) + + +platform.system = cross_system +platform.uname = cross_uname +platform.ios_ver = cross_ios_ver + + +########################################################################### +# sysconfig module patches +########################################################################### + +def cross_get_platform(): + return "{{platform}}-{{version_min}}-{{arch}}-{{sdk}}" + + +def cross_get_sysconfigdata_name(): + return "_sysconfigdata__{{platform}}_{{arch}}-{{sdk}}" + + +sysconfig.get_platform = cross_get_platform +sysconfig._get_sysconfigdata_name = cross_get_sysconfigdata_name + +# Force sysconfig data to be loaded (and cached). +sysconfig._init_config_vars() diff --git a/patch/Python/_cross_venv.py b/patch/Python/_cross_venv.py new file mode 100644 index 0000000..a51e5cb --- /dev/null +++ b/patch/Python/_cross_venv.py @@ -0,0 +1,103 @@ +import shutil +import sys +import sysconfig +from pathlib import Path + +SITE_PACKAGE_PATH = Path(__file__).parent + +########################################################################### +# importlib module patches +########################################################################### + + +def patch_env_create(env): + """ + Patch the process of creating virtual environments to ensure that the cross + environment modification files are also copied as part of environment + creation. + """ + old_pip_env_create = env._PipBackend.create + + def pip_env_create(self, path, *args, **kwargs): + result = old_pip_env_create(self, path, *args, **kwargs) + # Copy any _cross_*.pth or _cross_*.py file, plus the cross-platform + # sysconfigdata module to the new environment. + data_name = sysconfig._get_sysconfigdata_name() + for filename in [ + "_cross_venv.pth", + "_cross_venv.py", + f"_cross_{sys.implementation._multiarch.replace('-', '_')}.py", + f"{data_name}.py", + ]: + src = SITE_PACKAGE_PATH / filename + target = Path(path) / src.relative_to( + SITE_PACKAGE_PATH.parent.parent.parent + ) + if not target.exists(): + shutil.copy(src, target) + return result + + env._PipBackend.create = pip_env_create + + +# Import hook that patches the creation of virtual environments by `build` +# +# The approach used here is the same as the one used by virtualenv to patch +# distutils (but without support for the older load_module API). +# https://docs.python.org/3/library/importlib.html#setting-up-an-importer +_BUILD_PATCH = ("build.env",) + + +class _Finder: + """A meta path finder that allows patching the imported build modules.""" + + fullname = None + + # lock[0] is threading.Lock(), but initialized lazily to avoid importing + # threading very early at startup, because there are gevent-based + # applications that need to be first to import threading by themselves. + # See https://github.com/pypa/virtualenv/issues/1895 for details. + lock = [] # noqa: RUF012 + + def find_spec(self, fullname, path, target=None): + if fullname in _BUILD_PATCH and self.fullname is None: + # initialize lock[0] lazily + if len(self.lock) == 0: + import threading + + lock = threading.Lock() + # there is possibility that two threads T1 and T2 are + # simultaneously running into find_spec, observing .lock as + # empty, and further going into hereby initialization. However + # due to the GIL, list.append() operation is atomic and this + # way only one of the threads will "win" to put the lock + # - that every thread will use - into .lock[0]. + # https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe + self.lock.append(lock) + + from functools import partial + from importlib.util import find_spec + + with self.lock[0]: + self.fullname = fullname + try: + spec = find_spec(fullname, path) + if spec is not None: + # https://www.python.org/dev/peps/pep-0451/#how-loading-will-work + old = spec.loader.exec_module + func = self.exec_module + if old is not func: + spec.loader.exec_module = partial(func, old) + return spec + finally: + self.fullname = None + return None + + @staticmethod + def exec_module(old, module): + old(module) + if module.__name__ in _BUILD_PATCH: + patch_env_create(module) + + +sys.meta_path.insert(0, _Finder()) diff --git a/patch/Python/make_cross_venv.py b/patch/Python/make_cross_venv.py new file mode 100644 index 0000000..7eb9da2 --- /dev/null +++ b/patch/Python/make_cross_venv.py @@ -0,0 +1,123 @@ +import pprint +import shutil +import sys +from pathlib import Path +from importlib import util as importlib_util + + +def localized_vars(orig_vars, slice_path): + """Update (where possible) any references to build-time variables with the + best guess of the installed location. + """ + # The host's sysconfigdata will include references to build-time variables. + # Update these to refer to the current known install location. + orig_prefix = orig_vars["prefix"] + localized_vars = {} + for key, value in orig_vars.items(): + final = value + if isinstance(value, str): + # Replace any reference to the build installation prefix + final = final.replace(orig_prefix, str(slice_path)) + # Replace any reference to the build-time Framework location + final = final.replace("-F .", f"-F {slice_path}") + localized_vars[key] = final + + return localized_vars + + +def localize_sysconfigdata(platform_config_path, venv_site_packages): + """Localize a sysconfigdata python module. + + :param platform_config_path: The platform config that contains the + sysconfigdata module to localize. + :param venv_site_packages: The site packages folder where the localized + sysconfigdata module should be output. + """ + # Find the "_sysconfigdata_*.py" file in the platform config + sysconfigdata_path = next(platform_config_path.glob("_sysconfigdata_*.py")) + + # Import the sysconfigdata module + spec = importlib_util.spec_from_file_location( + sysconfigdata_path.stem, + sysconfigdata_path + ) + if spec is None: + msg = f"Unable to load spec for {sysconfigdata_path}" + raise ValueError(msg) + if spec.loader is None: + msg = f"Spec for {sysconfigdata_path} does not define a loader" + raise ValueError(msg) + sysconfigdata = importlib_util.module_from_spec(spec) + spec.loader.exec_module(sysconfigdata) + + # Write the updated sysconfigdata module into the cross-platform site. + slice_path = sysconfigdata_path.parent.parent.parent + with (venv_site_packages / sysconfigdata_path.name).open("w") as f: + f.write(f"# Generated from {sysconfigdata_path}\n") + f.write("build_time_vars = ") + pprint.pprint( + localized_vars(sysconfigdata.build_time_vars, slice_path), + stream=f, + compact=True + ) + + +def make_cross_venv(venv_path: Path, platform_config_path: Path): + """Convert a virtual environment into a cross-platform environment. + + :param venv_path: The path to the root of the venv. + :param platform_config_path: The path containing the platform config. + """ + if not venv_path.exists(): + raise ValueError(f"Virtual environment {venv_path} does not exist.") + if not (venv_path / "bin/python3").exists(): + raise ValueError(f"{venv_path} does not appear to be a virtual environment.") + + print( + f"Converting {venv_path} into a {platform_config_path.name} environment... ", + end="", + ) + + LIB_PATH = f"lib/python{sys.version_info[0]}.{sys.version_info[1]}" + + # Update path references in the sysconfigdata to reflect local conditions. + venv_site_packages = venv_path / LIB_PATH / "site-packages" + localize_sysconfigdata(platform_config_path, venv_site_packages) + + # Copy in the site-package environment modifications. + cross_multiarch = f"_cross_{platform_config_path.name.replace('-', '_')}" + shutil.copy( + platform_config_path / f"{cross_multiarch}.py", + venv_site_packages / f"{cross_multiarch}.py", + ) + shutil.copy( + platform_config_path / "_cross_venv.py", + venv_site_packages / "_cross_venv.py", + ) + # Write the .pth file that will enable the cross-env modifications + (venv_site_packages / "_cross_venv.pth").write_text( + f"import {cross_multiarch}; import _cross_venv\n" + ) + + print("done.") + + +if __name__ == "__main__": + try: + platform_config_path = Path(sys.argv[2]).resolve() + except IndexError: + platform_config_path = Path(__file__).parent + + try: + venv_path = Path(sys.argv[1]).resolve() + make_cross_venv(venv_path, platform_config_path) + except IndexError: + print(""" +Convert a virtual environment in to a cross-platform environment. + +Usage: + make_cross_venv () + +If an explicit platform config isn't provided, it is assumed the directory +containing the make_cross_venv script *is* a platform config. +""") diff --git a/patch/Python/module.modulemap.prefix b/patch/Python/module.modulemap.prefix new file mode 100644 index 0000000..e3b3aaf --- /dev/null +++ b/patch/Python/module.modulemap.prefix @@ -0,0 +1,20 @@ +module Python { + umbrella header "Python.h" + export * + link "Python" + + exclude header "datetime.h" + exclude header "dynamic_annotations.h" + exclude header "errcode.h" + exclude header "frameobject.h" + exclude header "marshal.h" + exclude header "opcode_ids.h" + exclude header "opcode.h" + exclude header "osdefs.h" + exclude header "py_curses.h" + exclude header "pyconfig-arm32_64.h" + exclude header "pyconfig-arm64.h" + exclude header "pyconfig-x86_64.h" + exclude header "pydtrace.h" + exclude header "pyexpat.h" + exclude header "structmember.h" diff --git a/patch/Python/sitecustomize.iOS.py b/patch/Python/sitecustomize.iOS.py deleted file mode 100644 index ccc291f..0000000 --- a/patch/Python/sitecustomize.iOS.py +++ /dev/null @@ -1,114 +0,0 @@ -# A site customization that can be used to trick pip into installing -# packages cross-platform. If the folder containing this file is on -# your PYTHONPATH when you invoke pip, pip will behave as if it were -# running on {{os}}. -import collections -import distutils.ccompiler -import distutils.unixccompiler -import os -import platform -import sys -import sysconfig -import types - -# Make platform.system() return "{{os}}" -def custom_system(): - return "{{os}}" - -platform.system = custom_system - -# Make platform.ios_ver() return an appropriate namedtuple -IOSVersionInfo = collections.namedtuple( - "IOSVersionInfo", - ["system", "release", "model", "is_simulator"] -) - -def custom_ios_ver(system="", release="", model="", is_simulator=False): - return IOSVersionInfo("{{os}}", "{{version_min}}", "iPhone", {{is_simulator}}) - -platform.ios_ver = custom_ios_ver - -# Make sys.implementation._multiarch return the multiarch description -sys.implementation._multiarch = "{{multiarch}}" - -# Make sysconfig.get_platform() return the platform tag -def custom_get_platform(): - return "{{tag}}" - -sysconfig.get_platform = custom_get_platform - -# Make distutils raise errors if you try to use it to build modules. -DISABLED_COMPILER_ERROR = "Cannot compile native modules" - -distutils.ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled" -distutils.ccompiler.compiler_class["disabled"] = ( - "disabledcompiler", - "DisabledCompiler", - "Compiler disabled ({})".format(DISABLED_COMPILER_ERROR), -) - - -def disabled_compiler(prefix): - # No need to give any more advice here: that will come from the higher-level code in pip. - from distutils.errors import DistutilsPlatformError - - raise DistutilsPlatformError("{}: {}".format(prefix, DISABLED_COMPILER_ERROR)) - - -class DisabledCompiler(distutils.ccompiler.CCompiler): - compiler_type = "disabled" - - def preprocess(*args, **kwargs): - disabled_compiler("CCompiler.preprocess") - - def compile(*args, **kwargs): - disabled_compiler("CCompiler.compile") - - def create_static_lib(*args, **kwargs): - disabled_compiler("CCompiler.create_static_lib") - - def link(*args, **kwargs): - disabled_compiler("CCompiler.link") - - -# To maximize the chance of the build getting as far as actually calling compile(), make -# sure the class has all of the expected attributes. -for name in [ - "src_extensions", - "obj_extension", - "static_lib_extension", - "shared_lib_extension", - "static_lib_format", - "shared_lib_format", - "exe_extension", -]: - setattr( - DisabledCompiler, name, getattr(distutils.unixccompiler.UnixCCompiler, name) - ) - -DisabledCompiler.executables = { - name: [DISABLED_COMPILER_ERROR.replace(" ", "_")] - for name in distutils.unixccompiler.UnixCCompiler.executables -} - -disabled_mod = types.ModuleType("distutils.disabledcompiler") -disabled_mod.DisabledCompiler = DisabledCompiler -sys.modules["distutils.disabledcompiler"] = disabled_mod - - -# Try to disable native builds for packages which don't use the distutils native build -# system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl). -for tool in ["ar", "as", "cc", "cxx", "ld"]: - os.environ[tool.upper()] = DISABLED_COMPILER_ERROR.replace(" ", "_") - - -# Call the next sitecustomize script if there is one -# (https://nedbatchelder.com/blog/201001/running_code_at_python_startup.html). -del sys.modules["sitecustomize"] -this_dir = os.path.dirname(__file__) -path_index = sys.path.index(this_dir) -del sys.path[path_index] -try: - import sitecustomize # noqa: F401 -finally: - sys.path.insert(path_index, this_dir) diff --git a/patch/Python/sitecustomize.macOS.py b/patch/Python/sitecustomize.macOS.py deleted file mode 100644 index 500714d..0000000 --- a/patch/Python/sitecustomize.macOS.py +++ /dev/null @@ -1,14 +0,0 @@ -# A site customization that can be used to trick pip into installing -# packages cross-platform. If the folder containing this file is on -# your PYTHONPATH when you invoke pip, pip will behave as if it were -# running on {{arch}}. -import platform - -# Make platform.mac_ver() return {{arch}} -orig_mac_ver = platform.mac_ver - -def custom_mac_ver(): - orig = orig_mac_ver() - return orig[0], orig[1], "{{arch}}" - -platform.mac_ver = custom_mac_ver diff --git a/patch/Python/sitecustomize.py.tmpl b/patch/Python/sitecustomize.py.tmpl new file mode 100644 index 0000000..0330575 --- /dev/null +++ b/patch/Python/sitecustomize.py.tmpl @@ -0,0 +1,22 @@ +# A site customization that can be used to trick pip into installing packages +# cross-platform. If the folder containing this file is on your PYTHONPATH when +# you invoke python, the interpreter will behave as if it were running on +# {{arch}} {{sdk}}. +import sys +import os + +# Apply the cross-platform patch +import _cross_{{arch}}_{{sdk}} +import _cross_venv + + +# Call the next sitecustomize script if there is one +# (https://nedbatchelder.com/blog/201001/running_code_at_python_startup.html). +del sys.modules["sitecustomize"] +this_dir = os.path.dirname(__file__) +path_index = sys.path.index(this_dir) +del sys.path[path_index] +try: + import sitecustomize # noqa: F401 +finally: + sys.path.insert(path_index, this_dir) diff --git a/patch/Python/sitecustomize.tvOS.py b/patch/Python/sitecustomize.tvOS.py deleted file mode 100644 index d7d86e3..0000000 --- a/patch/Python/sitecustomize.tvOS.py +++ /dev/null @@ -1,99 +0,0 @@ -# A site customization that can be used to trick pip into installing -# packages cross-platform. If the folder containing this file is on -# your PYTHONPATH when you invoke pip, pip will behave as if it were -# running on {{os}}. -import distutils.ccompiler -import distutils.unixccompiler -import os -import platform -import sys -import sysconfig -import types - -# Make platform.system() return "{{os}}" -def custom_system(): - return "{{os}}" - -platform.system = custom_system - -# Make sysconfig.get_platform() return "{{tag}}" -def custom_get_platform(): - return "{{tag}}" - -sysconfig.get_platform = custom_get_platform - -# Make distutils raise errors if you try to use it to build modules. -DISABLED_COMPILER_ERROR = "Cannot compile native modules" - -distutils.ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled" -distutils.ccompiler.compiler_class["disabled"] = ( - "disabledcompiler", - "DisabledCompiler", - "Compiler disabled ({})".format(DISABLED_COMPILER_ERROR), -) - - -def disabled_compiler(prefix): - # No need to give any more advice here: that will come from the higher-level code in pip. - from distutils.errors import DistutilsPlatformError - - raise DistutilsPlatformError("{}: {}".format(prefix, DISABLED_COMPILER_ERROR)) - - -class DisabledCompiler(distutils.ccompiler.CCompiler): - compiler_type = "disabled" - - def preprocess(*args, **kwargs): - disabled_compiler("CCompiler.preprocess") - - def compile(*args, **kwargs): - disabled_compiler("CCompiler.compile") - - def create_static_lib(*args, **kwargs): - disabled_compiler("CCompiler.create_static_lib") - - def link(*args, **kwargs): - disabled_compiler("CCompiler.link") - - -# To maximize the chance of the build getting as far as actually calling compile(), make -# sure the class has all of the expected attributes. -for name in [ - "src_extensions", - "obj_extension", - "static_lib_extension", - "shared_lib_extension", - "static_lib_format", - "shared_lib_format", - "exe_extension", -]: - setattr( - DisabledCompiler, name, getattr(distutils.unixccompiler.UnixCCompiler, name) - ) - -DisabledCompiler.executables = { - name: [DISABLED_COMPILER_ERROR.replace(" ", "_")] - for name in distutils.unixccompiler.UnixCCompiler.executables -} - -disabled_mod = types.ModuleType("distutils.disabledcompiler") -disabled_mod.DisabledCompiler = DisabledCompiler -sys.modules["distutils.disabledcompiler"] = disabled_mod - - -# Try to disable native builds for packages which don't use the distutils native build -# system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl). -for tool in ["ar", "as", "cc", "cxx", "ld"]: - os.environ[tool.upper()] = DISABLED_COMPILER_ERROR.replace(" ", "_") - - -# Call the next sitecustomize script if there is one -# (https://nedbatchelder.com/blog/201001/running_code_at_python_startup.html). -del sys.modules["sitecustomize"] -this_dir = os.path.dirname(__file__) -path_index = sys.path.index(this_dir) -del sys.path[path_index] -try: - import sitecustomize # noqa: F401 -finally: - sys.path.insert(path_index, this_dir) diff --git a/patch/Python/sitecustomize.watchOS.py b/patch/Python/sitecustomize.watchOS.py deleted file mode 100644 index d7d86e3..0000000 --- a/patch/Python/sitecustomize.watchOS.py +++ /dev/null @@ -1,99 +0,0 @@ -# A site customization that can be used to trick pip into installing -# packages cross-platform. If the folder containing this file is on -# your PYTHONPATH when you invoke pip, pip will behave as if it were -# running on {{os}}. -import distutils.ccompiler -import distutils.unixccompiler -import os -import platform -import sys -import sysconfig -import types - -# Make platform.system() return "{{os}}" -def custom_system(): - return "{{os}}" - -platform.system = custom_system - -# Make sysconfig.get_platform() return "{{tag}}" -def custom_get_platform(): - return "{{tag}}" - -sysconfig.get_platform = custom_get_platform - -# Make distutils raise errors if you try to use it to build modules. -DISABLED_COMPILER_ERROR = "Cannot compile native modules" - -distutils.ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled" -distutils.ccompiler.compiler_class["disabled"] = ( - "disabledcompiler", - "DisabledCompiler", - "Compiler disabled ({})".format(DISABLED_COMPILER_ERROR), -) - - -def disabled_compiler(prefix): - # No need to give any more advice here: that will come from the higher-level code in pip. - from distutils.errors import DistutilsPlatformError - - raise DistutilsPlatformError("{}: {}".format(prefix, DISABLED_COMPILER_ERROR)) - - -class DisabledCompiler(distutils.ccompiler.CCompiler): - compiler_type = "disabled" - - def preprocess(*args, **kwargs): - disabled_compiler("CCompiler.preprocess") - - def compile(*args, **kwargs): - disabled_compiler("CCompiler.compile") - - def create_static_lib(*args, **kwargs): - disabled_compiler("CCompiler.create_static_lib") - - def link(*args, **kwargs): - disabled_compiler("CCompiler.link") - - -# To maximize the chance of the build getting as far as actually calling compile(), make -# sure the class has all of the expected attributes. -for name in [ - "src_extensions", - "obj_extension", - "static_lib_extension", - "shared_lib_extension", - "static_lib_format", - "shared_lib_format", - "exe_extension", -]: - setattr( - DisabledCompiler, name, getattr(distutils.unixccompiler.UnixCCompiler, name) - ) - -DisabledCompiler.executables = { - name: [DISABLED_COMPILER_ERROR.replace(" ", "_")] - for name in distutils.unixccompiler.UnixCCompiler.executables -} - -disabled_mod = types.ModuleType("distutils.disabledcompiler") -disabled_mod.DisabledCompiler = DisabledCompiler -sys.modules["distutils.disabledcompiler"] = disabled_mod - - -# Try to disable native builds for packages which don't use the distutils native build -# system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl). -for tool in ["ar", "as", "cc", "cxx", "ld"]: - os.environ[tool.upper()] = DISABLED_COMPILER_ERROR.replace(" ", "_") - - -# Call the next sitecustomize script if there is one -# (https://nedbatchelder.com/blog/201001/running_code_at_python_startup.html). -del sys.modules["sitecustomize"] -this_dir = os.path.dirname(__file__) -path_index = sys.path.index(this_dir) -del sys.path[path_index] -try: - import sitecustomize # noqa: F401 -finally: - sys.path.insert(path_index, this_dir)