Python 3.12 (#3)

* git * Add limited Python 3.12 support * add 3.12.2 * run ci on 3.12.2 * typo * restore original testcapi * skip tkinter for 3.12 * roll forward patches * build libffi in ci test * fix ci * Revert "restore original testcapi" This reverts commit 6ac3063. * update testcapi sources * try installing libffi to system * set cflags * go back to deps directory * increase sdk level * update deepfreeze.py reference * build _tokenize module * fix syntax * use full path to tokenize * add _typing * add Rpcrt4 for win _socket * update c standard selection * add to _tokenize sources * more _tokenize sources * don't compile some builtins separately * try treating _tokenize as builtin * Revert "try treating _tokenize as builtin" This reverts commit 7a7c738. * Revert "don't compile some builtins separately" This reverts commit 8ac6161. * Revert "Revert "don't compile some builtins separately"" This reverts commit cae6053. * build shared libpython * Revert "build shared libpython" This reverts commit 3bcb4a9. * Revert "Revert "Revert "don't compile some builtins separately""" This reverts commit 9f3b647. * some platform-specific tweaks * always build _tokenize _typing as builtin * add 3.12.0 3.12.1 --------- Co-authored-by: Minmin Gong <[email protected]>
bjia56 · Mar 15, 2024 · 8f39f0a · 8f39f0a
1 parent 0996ade
commit 8f39f0a
Show file tree

Hide file tree

Showing 12 changed files with 404 additions and 28 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -12,7 +12,7 @@ on:
         type: boolean
 
 env:
-  MACOSX_DEPLOYMENT_TARGET: 10.5
+  MACOSX_DEPLOYMENT_TARGET: 10.9
 
 jobs:
   tests:
@@ -21,7 +21,7 @@ jobs:
       fail-fast: true
       matrix:
         runs-on: [macos-latest]
-        python-version: [3.8.18, 3.9.18, 3.10.13, 3.11.8]
+        python-version: [3.9.18, 3.10.13, 3.11.8, 3.12.2]
 
     name: python-${{ matrix.python-version }}-${{ matrix.runs-on }}
     steps:
@@ -35,16 +35,29 @@ jobs:
     - name: Install tools
       run: |
         brew install gpatch
+  
+    - name: Build libffi
+      if: ${{ startsWith(matrix.python-version, '3.12') }}
+      run: |
+        mkdir -p ${{ github.workspace }}/deps
+        wget -q https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz
+        tar -xf libffi-3.4.2.tar.gz
+        cd libffi-3.4.2
+        ./configure --prefix ${{ github.workspace }}/deps
+        make -j4
+        make install
 
     - name: Build
       run: |
         mkdir -p python-build && mkdir -p python-install
         cd python-build
-        cmake \
+        CFLAGS="-I${{ github.workspace }}/deps/include" cmake \
           ${{ inputs.verbose && '--trace-expand --debug-find' || '' }} \
           -DCMAKE_INSTALL_PREFIX:PATH=${{ github.workspace }}/python-install \
           -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ env.MACOSX_DEPLOYMENT_TARGET }} \
-          ${{ startsWith(matrix.python-version, '3.11') && '-DUSE_SYSTEM_TCL=OFF' || '' }} \
+          ${{ startsWith(matrix.python-version, '3.11') && '-DUSE_SYSTEM_TCL=OFF' || '' }} ${{ startsWith(matrix.python-version, '3.12') && '-DUSE_SYSTEM_TCL=OFF' || '' }} \
+          ${{ startsWith(matrix.python-version, '3.12') && format('-DLibFFI_INCLUDE_DIR:PATH={0}/deps/include', github.workspace) || '' }} \
+          ${{ startsWith(matrix.python-version, '3.12') && format('-DLibFFI_LIBRARY:FILEPATH={0}/deps/lib/libffi.a', github.workspace) || '' }} \
           -DPYTHON_VERSION=${{ matrix.python-version }} \
           ..
         make -j4 ${{ inputs.verbose && 'VERBOSE=1' || '' }}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -291,6 +291,10 @@ set(_download_3.10.13_md5 "cbcad7f5e759176bf8ce8a5f9d487087")
 # 3.11.x
 set(_download_3.11.7_md5 "ef61f81ec82c490484219c7f0ec96783")
 set(_download_3.11.8_md5 "7fb0bfaa2f6aae4aadcdb51abe957825")
+# 3.12.x
+set(_download_3.12.0_md5 "d6eda3e1399cef5dfde7c4f319b0596c")
+set(_download_3.12.1_md5 "51c5c22dcbc698483734dff5c8028606")
+set(_download_3.12.2_md5 "4e64a004f8ad9af1a75607cfd0d5a8c8")
 
 set(_extracted_dir "Python-${PY_VERSION}")
 
@@ -383,10 +387,7 @@ if(WIN32 AND IS_PY3)
 endif()
 
 if(PY_VERSION VERSION_GREATER_EQUAL "3.11")
-  set(CMAKE_C_STANDARD "11")
   find_package(Python3 COMPONENTS Interpreter)
-else()
-  set(CMAKE_C_STANDARD "99")
 endif()
 
 # Options depending of the python version
@@ -559,7 +560,9 @@ include_directories(${SRC_DIR}/Python)
 
 # Set cflags used by all components
 if(CMAKE_C_COMPILER_ID MATCHES GNU)
-    if(PY_VERSION VERSION_GREATER_EQUAL "3.6")
+    if(PY_VERSION VERSION_GREATER_EQUAL "3.11")
+        append_if_absent(CMAKE_C_FLAGS "-std=c11")
+    elseif(PY_VERSION VERSION_GREATER_EQUAL "3.6")
         append_if_absent(CMAKE_C_FLAGS "-std=c99")
     endif()
     append_if_absent(CMAKE_C_FLAGS "-Wall")

diff --git a/cmake/config-mingw/pyconfig.h b/cmake/config-mingw/pyconfig.h
@@ -49,6 +49,14 @@ WIN32 is still required for the locale module.
 #define USE_SOCKET
 #endif
 
+#if PY_VERSION_HEX >= 0x030C0000
+#  define WIN32_LEAN_AND_MEAN
+#  include <windows.h>
+
+#  define MS_WINDOWS_DESKTOP
+#  define HAVE_WINDOWS_CONSOLE_IO 1
+#endif
+
 /* ------------------------------------------------------------------------*/
 /* (i686|x86_64)-w64-mingw32 toolchains defines __MINGW32__ */
 #ifndef __MINGW32__
@@ -140,6 +148,9 @@ WIN32 is still required for the locale module.
 #define HAVE_TMPNAM
 #define HAVE_CLOCK
 #define HAVE_STRERROR
+#if PY_VERSION_HEX >= 0x030C0000
+#  define HAVE_CLOCK_GETTIME 1
+#endif
 
 #include <io.h>
 #include <stdio.h>
@@ -482,6 +493,11 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
 /* Define to 1 if you have the <process.h> header file. */
 #define HAVE_PROCESS_H 1
 
+#if PY_VERSION_HEX >= 0x030C0000
+/* Define to 1 if you have the <pthread.h> header file. */
+#define HAVE_PTHREAD_H
+#endif
+
 /* Define to 1 if you have the <signal.h> header file. */
 #define HAVE_SIGNAL_H 1
 

diff --git a/cmake/extensions/CMakeLists.txt b/cmake/extensions/CMakeLists.txt
@@ -78,7 +78,47 @@ endif()
 add_python_extension(_random ${WIN32_BUILTIN} SOURCES _randommodule.c)
 add_python_extension(strop ${WIN32_BUILTIN} REQUIRES IS_PY2 SOURCES stropmodule.c)
 add_python_extension(_struct ${WIN32_BUILTIN} SOURCES _struct.c)
-add_python_extension(_testcapi NEVER_BUILTIN SOURCES _testcapimodule.c)
+set(testcapi_SOURCES _testcapimodule.c)
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+  list(APPEND testcapi_SOURCES
+    _testcapi/abstract.c
+    _testcapi/buffer.c
+    _testcapi/bytearray.c
+    _testcapi/bytes.c
+    _testcapi/code.c
+    _testcapi/codec.c
+    _testcapi/complex.c
+    _testcapi/datetime.c
+    _testcapi/dict.c
+    _testcapi/docstring.c
+    _testcapi/exceptions.c
+    _testcapi/file.c
+    _testcapi/float.c
+    _testcapi/gc.c
+    _testcapi/getargs.c
+    _testcapi/heaptype.c
+    _testcapi/heaptype_relative.c
+    _testcapi/immortal.c
+    _testcapi/list.c
+    _testcapi/long.c
+    _testcapi/mem.c
+    _testcapi/numbers.c
+    _testcapi/pyos.c
+    _testcapi/pytime.c
+    _testcapi/set.c
+    _testcapi/structmember.c
+    _testcapi/sys.c
+    _testcapi/tuple.c
+    _testcapi/unicode.c
+    _testcapi/vectorcall.c
+    _testcapi/vectorcall_limited.c
+    _testcapi/watchers.c
+  )
+endif()
+if(PY_VERSION VERSION_LESS "3.12" OR UNIX)
+    # This module seems to build incorrectly for Python 3.12 Windows
+    add_python_extension(_testcapi NEVER_BUILTIN SOURCES ${testcapi_SOURCES})
+endif()
 set(thread2_NAME thread)
 set(thread3_NAME _thread)
 set(thread2_SOURCES ${SRC_DIR}/Modules/threadmodule.c)
@@ -144,7 +184,9 @@ if(PY_VERSION VERSION_GREATER_EQUAL "3.6")
         _blake2/blake2s_impl.c
     )
     add_python_extension(_blake2 ${WIN32_BUILTIN} SOURCES ${_blake2_SOURCES})
-    add_python_extension(_sha3 ${WIN32_BUILTIN} SOURCES _sha3/sha3module.c)
+    if(PY_VERSION VERSION_LESS "3.12")
+        add_python_extension(_sha3 ${WIN32_BUILTIN} SOURCES _sha3/sha3module.c)
+    endif()
     add_python_extension(_testconsole ${WIN32_BUILTIN} REQUIRES WIN32 SOURCES ../PC/_testconsole.c)
 endif()
 
@@ -169,6 +211,11 @@ if(PY_VERSION VERSION_EQUAL "3.9")
     add_python_extension(_peg_parser ALWAYS_BUILTIN SOURCES _peg_parser.c)
 endif()
 
+# Python 3.12
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    add_python_extension(_testsinglephase BUILTIN SOURCES _testsinglephase.c)
+endif()
+
 # UNIX-only extensions
 add_python_extension(fcntl REQUIRES UNIX SOURCES fcntlmodule.c)
 add_python_extension(grp REQUIRES UNIX SOURCES grpmodule.c)
@@ -254,6 +301,13 @@ if(EXISTS ${module_src})
 endif()
 add_python_extension(_distutils_findvs REQUIRES WIN32 IS_PY3 HAS_DISTUTILS_FINDVS_MODULE_SRC SOURCES ${module_src})
 
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    add_python_extension(_wmi REQUIRES WIN32 IS_PY3
+        SOURCES ${SRC_DIR}/PC/_wmimodule.cpp
+        LIBRARIES propsys wbemuuid
+    )
+endif()
+
 # Multiprocessing is different on unix and windows
 if(UNIX)
     set(_multiprocessing2_SOURCES
@@ -305,7 +359,7 @@ if(WIN32)
         DEFINITIONS EAI_ADDRFAMILY
                     HAVE_SOCKADDR_STORAGE
                     HAVE_ADDRINFO
-        LIBRARIES ws2_32 ${M_LIBRARIES} $<$<VERSION_GREATER_EQUAL:${PY_VERSION},3.8>:iphlpapi>
+        LIBRARIES ws2_32 ${M_LIBRARIES} $<$<VERSION_GREATER_EQUAL:${PY_VERSION},3.8>:iphlpapi> $<$<VERSION_GREATER_EQUAL:${PY_VERSION},3.12>:Rpcrt4>
     )
 else()
     if(PY_VERSION VERSION_GREATER_EQUAL "2.7.4" AND PY_VERSION VERSION_LESS "3")
@@ -389,15 +443,21 @@ if(WIN32)
 else()
     if(APPLE)
         if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64")
+            set(ctypes_SOURCES
+                _ctypes/malloc_closure.c)
+            if(PY_VERSION VERSION_LESS "3.12")
+                list(APPEND ctypes_SOURCES
+                    _ctypes/darwin/dlfcn_simple.c
+                    _ctypes/libffi_osx/ffi.c
+                    _ctypes/libffi_osx/x86/darwin64.S
+                    _ctypes/libffi_osx/x86/x86-darwin.S
+                    _ctypes/libffi_osx/x86/x86-ffi_darwin.c
+                    _ctypes/libffi_osx/x86/x86-ffi64.c
+                )
+            endif()
             add_python_extension(_ctypes
                 SOURCES ${ctypes_COMMON_SOURCES}
-                        _ctypes/malloc_closure.c
-                        _ctypes/darwin/dlfcn_simple.c
-                        _ctypes/libffi_osx/ffi.c
-                        _ctypes/libffi_osx/x86/darwin64.S
-                        _ctypes/libffi_osx/x86/x86-darwin.S
-                        _ctypes/libffi_osx/x86/x86-ffi_darwin.c
-                        _ctypes/libffi_osx/x86/x86-ffi64.c
+                        ${ctypes_SOURCES}
                 INCLUDEDIRS ${SRC_DIR}/Modules/_ctypes/libffi_osx/include
                             ${SRC_DIR}/Modules/_ctypes/darwin
                 DEFINITIONS MACOSX
@@ -643,16 +703,28 @@ add_python_extension(_sha512 REQUIRES HASH_NOT_AVAILABLE ${WIN32_BUILTIN} SOURCE
 
 else()
 
-# We always compile these even when OpenSSL is available (issue #14693).
-# It's harmless and the object code is tiny (40-50 KB per module,
-# only loaded when actually used).
-add_python_extension(_md5 ${WIN32_BUILTIN} SOURCES md5module.c)
-add_python_extension(_sha1 ${WIN32_BUILTIN} SOURCES sha1module.c)
-add_python_extension(_sha256 ${WIN32_BUILTIN} SOURCES sha256module.c)
-add_python_extension(_sha512 ${WIN32_BUILTIN} SOURCES sha512module.c)
+if(PY_VERSION VERSION_LESS "3.12")
+    # We always compile these even when OpenSSL is available (issue #14693).
+    # It's harmless and the object code is tiny (40-50 KB per module,
+    # only loaded when actually used).
+    add_python_extension(_md5 ${WIN32_BUILTIN} SOURCES md5module.c)
+    add_python_extension(_sha1 ${WIN32_BUILTIN} SOURCES sha1module.c)
+    add_python_extension(_sha256 ${WIN32_BUILTIN} SOURCES sha256module.c)
+    add_python_extension(_sha512 ${WIN32_BUILTIN} SOURCES sha512module.c)
+endif()
 
 endif()
 
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    add_python_extension(_tokenize
+        SOURCES ${SRC_DIR}/Python/Python-tokenize.c
+        ALWAYS_BUILTIN
+    )
+    add_python_extension(_typing
+        SOURCES _typingmodule.c
+        ALWAYS_BUILTIN
+    )
+endif()
 
 # Extensions that depend on other libraries
 set(binascii_REQUIRES "")

diff --git a/cmake/libpython/CMakeLists.txt b/cmake/libpython/CMakeLists.txt
@@ -142,11 +142,15 @@ if(MSVC)
 endif()
 
 set(OBJECT3_SOURCES
-    ${SRC_DIR}/Objects/accu.c
     ${SRC_DIR}/Objects/bytesobject.c
     ${SRC_DIR}/Objects/namespaceobject.c
     ${SRC_DIR}/Objects/odictobject.c
 )
+if(PY_VERSION VERSION_LESS "3.12")
+    list(APPEND OBJECT3_SOURCES
+        ${SRC_DIR}/Objects/accu.c
+    )
+endif()
 if(MSVC)
     list(APPEND OBJECT3_SOURCES
         ${SRC_DIR}/PC/invalid_parameter_handler.c
@@ -157,6 +161,11 @@ if(PY_VERSION VERSION_GREATER_EQUAL "3.7")
         ${SRC_DIR}/Objects/call.c
     )
 endif()
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    list(APPEND OBJECT3_SOURCES
+        ${SRC_DIR}/Objects/typevarobject.c
+    )
+endif()
 
 set(OBJECT_COMMON_SOURCES # Equivalent to OBJECT_OBJS in Makefile.pre
     ${OBJECT${PY_VERSION_MAJOR}_SOURCES}
@@ -360,6 +369,18 @@ if(PY_VERSION VERSION_GREATER_EQUAL "3.11")
         ${SRC_DIR}/Python/specialize.c
     )
 endif()
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    list(APPEND PYTHON_COMMON_SOURCES
+        ${SRC_DIR}/Python/assemble.c
+        ${SRC_DIR}/Python/ceval_gil.c
+        ${SRC_DIR}/Python/flowgraph.c
+        ${SRC_DIR}/Python/instrumentation.c
+        ${SRC_DIR}/Python/intrinsics.c
+        ${SRC_DIR}/Python/legacy_tracing.c
+        ${SRC_DIR}/Python/perf_trampoline.c
+        ${SRC_DIR}/Python/tracemalloc.c
+    )
+endif()
 
 if(UNIX)
     list(APPEND PYTHON_COMMON_SOURCES
@@ -784,10 +805,15 @@ else()
       ${SRC_DIR}/Modules/getpath.py
     )
 
+  if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    set(DEEPFREEZE_PY ${SRC_DIR}/Tools/build/deepfreeze.py)
+  else()
+    set(DEEPFREEZE_PY ${SRC_DIR}/Tools/scripts/deepfreeze.py)
+  endif()
   add_custom_command(
     OUTPUT ${LIBPYTHON_DEEPFREEZE_SOURCES}
     COMMAND
-      ${Python3_EXECUTABLE} ${SRC_DIR}/Tools/scripts/deepfreeze.py
+      ${Python3_EXECUTABLE} ${DEEPFREEZE_PY}
         "${SRC_DIR}/Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap"
         "${SRC_DIR}/Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external"
         "${SRC_DIR}/Python/frozen_modules/zipimport.h:zipimport"
@@ -813,7 +839,7 @@ else()
         "${SRC_DIR}/Python/frozen_modules/frozen_only.h:frozen_only"
         "-o" "${LIBPYTHON_DEEPFREEZE_SOURCES}"
     DEPENDS
-      ${SRC_DIR}/Tools/scripts/deepfreeze.py
+      ${DEEPFREEZE_PY}
       ${LIBPYTHON_FROZEN_SOURCES}
   )
 endif()
@@ -910,10 +936,26 @@ elseif(UNIX)
         ${SRC_DIR}/Python/frozen.c
     )
 endif()
+if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+    list(APPEND LIBPYTHON_SOURCES
+        ${SRC_DIR}/Modules/_hacl/Hacl_Hash_MD5.c
+        ${SRC_DIR}/Modules/_hacl/Hacl_Hash_SHA1.c
+        ${SRC_DIR}/Modules/_hacl/Hacl_Hash_SHA2.c
+        ${SRC_DIR}/Modules/_hacl/Hacl_Hash_SHA3.c
+        ${SRC_DIR}/Modules/_xxinterpchannelsmodule.c
+        ${SRC_DIR}/Modules/md5module.c
+        ${SRC_DIR}/Modules/sha1module.c
+        ${SRC_DIR}/Modules/sha2module.c
+        ${SRC_DIR}/Modules/sha3module.c
+    )
+endif()
 
 # Build python libraries
 function(add_libpython name type install component)
     add_library(${name} ${type} ${LIBPYTHON_SOURCES})
+    if(PY_VERSION VERSION_GREATER_EQUAL "3.12")
+        target_include_directories(${name} PRIVATE ${SRC_DIR}/Modules/_hacl/include)
+    endif()
     target_link_libraries(${name} ${LIBPYTHON_TARGET_LIBRARIES})
 
     if(MSVC)

diff --git a/patches/3.12/01-PC-config_minimal.patch b/patches/3.12/01-PC-config_minimal.patch
@@ -0,0 +1,17 @@
+diff --git a/PC/config_minimal.c b/PC/config_minimal.c
+index 928a4efd32..e541204c79 100644
+--- a/PC/config_minimal.c
++++ b/PC/config_minimal.c
+@@ -8,6 +8,12 @@
+ /* Define extern variables omitted from minimal builds */
+ void *PyWin_DLLhModule = NULL;
+
++#if !defined(MS_DLL_ID)
++# error MS_DLL_ID must be defined
++#endif
++
++const char *PyWin_DLLVersionString = MS_DLL_ID;
++
+
+ extern PyObject* PyInit_faulthandler(void);
+ extern PyObject* PyInit__tracemalloc(void);