From 6125a6199519861406533ba2c17dfb8e308664c2 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sat, 12 Oct 2024 14:52:51 -0400 Subject: [PATCH] Resolve verstamp bootstraping problem | Allow GitHub installs (#2349) --- .github/workflows/main.yml | 9 +- CHANGES.txt | 2 + README.md | 2 +- mypy.ini | 3 +- setup.py | 49 ++----- win32/Lib/_win32verstamp_pywin32ctypes.py | 164 ++++++++++++++++++++++ win32/Lib/win32verstamp.py | 9 +- win32/scripts/VersionStamp/bulkstamp.py | 43 +++--- 8 files changed, 217 insertions(+), 64 deletions(-) create mode 100644 win32/Lib/_win32verstamp_pywin32ctypes.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 015651077..09c98f48e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,7 @@ jobs: - name: Build and install run: | - python setup.py --skip-verstamp install --user + python setup.py install --user - name: Run tests # Run the tests directly from the source dir so support files (eg, .wav files etc) @@ -93,7 +93,7 @@ jobs: python .github\workflows\download-arm64-libs.py .\arm64libs - name: Build wheels - run: python setup.py --skip-verstamp build_ext -L .\arm64libs --plat-name win-arm64 build --plat-name win-arm64 bdist_wheel --plat-name win-arm64 + run: python setup.py build_ext -L .\arm64libs --plat-name win-arm64 build --plat-name win-arm64 bdist_wheel --plat-name win-arm64 - uses: actions/upload-artifact@v3 if: ${{ always() }} @@ -110,8 +110,9 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - # This job only needs to target the oldest supported version (black@stable supports Python >=3.8) - python-version: "3.8" + # This job only needs to target the oldest version supported by our checkers + # (black>=24.10.0 supports Python >=3.9) + python-version: "3.9" cache: pip cache-dependency-path: .github/workflows/main.yml - run: pip install clang-format pycln diff --git a/CHANGES.txt b/CHANGES.txt index eea52c8eb..7f46e93c9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -14,6 +14,8 @@ https://mhammond.github.io/pywin32_installers.html. Coming in build 308, as yet unreleased -------------------------------------- +* Allowed installs from source w/o having pywin32 pre-installed (for instance, from GitHub) (#2349, @Avasam) +* Restored version stamping of installed DLLs (#2349, @Avasam) * Fixed a circular import between `win32comext.axscript.client.framework` and `win32comext.axscript.client.error` (#2381, @Avasam) * Remove long-deprecated `win32com.server.dispatcher.DispatcherWin32dbg` (#2382, @Avasam) diff --git a/README.md b/README.md index a74de0c42..b03454e45 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ the builds. Build 306 was the last released with this process. * Update setup.py with the new build number. -* Execute `make.bat`, wait forever, test the artifacts. +* Execute `make_all.bat`, wait forever, test the artifacts. * Upload .whl artifacts to pypi - we do this before pushing the tag because they might be rejected for an invalid `README.md`. Done via `py -3.? -m twine upload dist/*XXX*.whl`. diff --git a/mypy.ini b/mypy.ini index 962e9a6cc..bd1cd4316 100644 --- a/mypy.ini +++ b/mypy.ini @@ -48,11 +48,10 @@ exclude = (?x)( [mypy-adsi.*,dde,exchange,mapi,perfmon,servicemanager,win32api,win32console,win32clipboard,win32comext.adsi.adsi,win32event,win32evtlog,win32file,win32gui,win32help,win32pdh,win32process,win32ras,win32security,win32service,win32trace,win32ui,win32uiole,win32wnet,_win32sysloader,_winxptheme] ignore_missing_imports = True -; verstamp is installed from win32verstamp.py called in setup.py ; Most of win32com re-exports win32comext ; Test is a local untyped module in win32comext.axdebug ; pywin32_system32 is an empty module created in setup.py to store dlls -[mypy-verstamp,win32com.*,Test,pywin32_system32] +[mypy-win32com.*,Test,pywin32_system32] ignore_missing_imports = True ; Distutils being removed from stdlib currently causes some issues on Python 3.12 diff --git a/setup.py b/setup.py index 0f481c2c0..8b77726ae 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ __doc__ = """This is a distutils setup-script for the pywin32 extensions. The canonical source of truth for supported versions and build environments -is [the github CI](https://github.com/mhammond/pywin32/tree/main/.github/workflows). +is [the GitHub CI](https://github.com/mhammond/pywin32/tree/main/.github/workflows). To build and install locally for testing etc, you need a build environment which is capable of building the version of Python you are targeting, then: @@ -62,12 +62,6 @@ ) print("Building pywin32", pywin32_version) -try: - sys.argv.remove("--skip-verstamp") - skip_verstamp = True -except ValueError: - skip_verstamp = False - try: this_file = __file__ except NameError: @@ -978,35 +972,18 @@ def link( # target. Do this externally to avoid suddenly dragging in the # modules needed by this process, and which we will soon try and # update. - # Further, we don't really want to use sys.executable, because that - # means the build environment must have a current pywin32 installed - # in every version, which is a bit of a burden only for this. - # So we assume the "default" Python version (ie, the version run by - # py.exe) has pywin32 installed. - # (This creates a chicken-and-egg problem though! We used to work around - # this by ignoring failure to verstamp, but that's easy to miss. So now - # allow --skip-verstamp on the cmdline - but if it's not there, the - # verstamp must work.) - if not skip_verstamp: - args = ["py.exe", "-m", "win32verstamp"] - args.append(f"--version={pywin32_version}") - args.append("--comments=https://github.com/mhammond/pywin32") - args.append(f"--original-filename={os.path.basename(output_filename)}") - args.append("--product=PyWin32") - if "-v" not in sys.argv: - args.append("--quiet") - args.append(output_filename) - try: - self.spawn(args) - except Exception: - print("** Failed to versionstamp the binaries.") - # py.exe is not yet available for windows-arm64 so version stamp will fail - # ignore it for now - if platform.machine() != "ARM64": - print( - "** If you want to skip this step, pass '--skip-verstamp' on the setup.py command-line" - ) - raise + args = [ + sys.executable, + # NOTE: On Python 3.7, all args must be str + str(Path(__file__).parent / "win32" / "Lib" / "win32verstamp.py"), + f"--version={pywin32_version}", + "--comments=https://github.com/mhammond/pywin32", + f"--original-filename={os.path.basename(output_filename)}", + "--product=PyWin32", + "--quiet" if "-v" not in sys.argv else "", + output_filename, + ] + self.spawn(args) # Work around bpo-36302/bpo-42009 - it sorts sources but this breaks # support for building .mc files etc :( diff --git a/win32/Lib/_win32verstamp_pywin32ctypes.py b/win32/Lib/_win32verstamp_pywin32ctypes.py new file mode 100644 index 000000000..b878809b7 --- /dev/null +++ b/win32/Lib/_win32verstamp_pywin32ctypes.py @@ -0,0 +1,164 @@ +""" +A pure-python re-implementation of methods used by win32verstamp. +This is to avoid a bootstraping problem where win32verstamp is used during build, +but requires an installation of pywin32 to be present. +We used to work around this by ignoring failure to verstamp, but that's easy to miss. + +Implementations adapted, simplified and typed from: +- https://github.com/enthought/pywin32-ctypes/blob/main/win32ctypes/core/ctypes/_util.py +- https://github.com/enthought/pywin32-ctypes/blob/main/win32ctypes/core/cffi/_resource.py +- https://github.com/enthought/pywin32-ctypes/blob/main/win32ctypes/pywin32/win32api.py + +--- + +(C) Copyright 2014 Enthought, Inc., Austin, TX +All right reserved. + +This file is open source software distributed according to the terms in +https://github.com/enthought/pywin32-ctypes/blob/main/LICENSE.txt +""" + +from __future__ import annotations + +from collections.abc import Iterable +from ctypes import FormatError, WinDLL, get_last_error +from ctypes.wintypes import ( + BOOL, + DWORD, + HANDLE, + LPCWSTR, + LPVOID, + WORD, +) +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ctypes import _NamedFuncPointer + + from _typeshed import ReadableBuffer + from typing_extensions import Literal, SupportsBytes, SupportsIndex + +kernel32 = WinDLL("kernel32", use_last_error=True) + +### +# https://github.com/enthought/pywin32-ctypes/blob/main/win32ctypes/core/ctypes/_util.py +### + + +def make_error(function: _NamedFuncPointer) -> OSError: + code = get_last_error() + exception = OSError() + exception.winerror = code + exception.function = function.__name__ + exception.strerror = FormatError(code).strip() + return exception + + +def check_null(result: int | None, function: _NamedFuncPointer, *_) -> int: + if result is None: + raise make_error(function) + return result + + +def check_false(result: int | None, function: _NamedFuncPointer, *_) -> Literal[True]: + if not bool(result): + raise make_error(function) + else: + return True + + +### +# https://github.com/enthought/pywin32-ctypes/blob/main/win32ctypes/core/cffi/_resource.py +### + +_BeginUpdateResource = kernel32.BeginUpdateResourceW +_BeginUpdateResource.argtypes = [LPCWSTR, BOOL] +_BeginUpdateResource.restype = HANDLE +_BeginUpdateResource.errcheck = check_null # type: ignore[assignment] # ctypes is badly typed + + +_EndUpdateResource = kernel32.EndUpdateResourceW +_EndUpdateResource.argtypes = [HANDLE, BOOL] +_EndUpdateResource.restype = BOOL +_EndUpdateResource.errcheck = check_false # type: ignore[assignment] # ctypes is badly typed + +_UpdateResource = kernel32.UpdateResourceW +_UpdateResource.argtypes = [HANDLE, LPCWSTR, LPCWSTR, WORD, LPVOID, DWORD] +_UpdateResource.restype = BOOL +_UpdateResource.errcheck = check_false # type: ignore[assignment] # ctypes is badly typed + + +### +# https://github.com/enthought/pywin32-ctypes/blob/main/win32ctypes/pywin32/win32api.py +### + +LANG_NEUTRAL = 0x00 + + +def BeginUpdateResource(filename: str, delete: bool): + """Get a handle that can be used by the :func:`UpdateResource`. + + Parameters + ---------- + fileName : str + The filename of the module to load. + delete : bool + When true all existing resources are deleted + + Returns + ------- + result : hModule + Handle of the resource. + + """ + return _BeginUpdateResource(filename, delete) + + +def EndUpdateResource(handle: int, discard: bool) -> None: + """End the update resource of the handle. + + Parameters + ---------- + handle : hModule + The handle of the resource as it is returned + by :func:`BeginUpdateResource` + + discard : bool + When True all writes are discarded. + + """ + _EndUpdateResource(handle, discard) + + +def UpdateResource( + handle: int, + type: str | int, + name: str | int, + data: Iterable[SupportsIndex] | SupportsIndex | SupportsBytes | ReadableBuffer, + language: int = LANG_NEUTRAL, +) -> None: + """Update a resource. + + Parameters + ---------- + handle : hModule + The handle of the resource file as returned by + :func:`BeginUpdateResource`. + + type : str | int + The type of resource to update. + + name : str | int + The name or Id of the resource to update. + + data : bytes-like + A bytes like object is expected. + + language : int + Language to use, default is LANG_NEUTRAL. + + """ + lp_data = bytes(data) + _UpdateResource( + handle, LPCWSTR(type), LPCWSTR(name), language, lp_data, len(lp_data) + ) diff --git a/win32/Lib/win32verstamp.py b/win32/Lib/win32verstamp.py index e9f8c5e45..7b211303d 100644 --- a/win32/Lib/win32verstamp.py +++ b/win32/Lib/win32verstamp.py @@ -1,12 +1,15 @@ -""" Stamp a Win32 binary with version information. -""" +"""Stamp a Win32 binary with version information.""" import glob import optparse import os import struct -from win32api import BeginUpdateResource, EndUpdateResource, UpdateResource +from _win32verstamp_pywin32ctypes import ( + BeginUpdateResource, + EndUpdateResource, + UpdateResource, +) VS_FFI_SIGNATURE = -17890115 # 0xFEEF04BD VS_FFI_STRUCVERSION = 0x00010000 diff --git a/win32/scripts/VersionStamp/bulkstamp.py b/win32/scripts/VersionStamp/bulkstamp.py index 3a039ebb8..89be6347d 100644 --- a/win32/scripts/VersionStamp/bulkstamp.py +++ b/win32/scripts/VersionStamp/bulkstamp.py @@ -33,11 +33,15 @@ import fnmatch import os import sys +from collections.abc import Mapping +from optparse import Values -import verstamp -import win32api - -numStamped = 0 +try: + import win32verstamp +except ModuleNotFoundError: + # If run with pywin32 not already installed + sys.path.append(os.path.abspath(__file__ + "/../../../Lib")) + import win32verstamp g_patterns = [ "*.dll", @@ -47,9 +51,9 @@ ] -def walk(arg, dirname, names): - global numStamped - vars, debug, descriptions = arg +def walk(vars: Mapping[str, str], debug, descriptions, dirname, names) -> int: + """Returns the number of stamped files.""" + numStamped = 0 for name in names: for pat in g_patterns: if fnmatch.fnmatch(name, pat): @@ -60,11 +64,14 @@ def walk(arg, dirname, names): name = base[:-2] + ext is_dll = ext.lower() != ".exe" if os.path.normcase(name) in descriptions: - desc = descriptions[os.path.normcase(name)] + description = descriptions[os.path.normcase(name)] try: - verstamp.stamp(vars, pathname, desc, is_dll=is_dll) + options = Values( + {**vars, "description": description, "dll": is_dll} + ) + win32verstamp.stamp(pathname, options) numStamped += 1 - except win32api.error as exc: + except OSError as exc: print( "Could not stamp", pathname, @@ -76,13 +83,14 @@ def walk(arg, dirname, names): else: print("WARNING: description not provided for:", name) # skip branding this - assume already branded or handled elsewhere + return numStamped # print("Stamped", pathname) def load_descriptions(fname, vars): - retvars = {} + retvars: dict[str, str] = {} descriptions = {} lines = open(fname, "r").readlines() @@ -118,9 +126,7 @@ def load_descriptions(fname, vars): return retvars, descriptions -def scan(build, root, desc, **custom_vars): - global numStamped - numStamped = 0 +def scan(build, root: str, desc, **custom_vars): try: build = int(build) except ValueError: @@ -135,10 +141,11 @@ def scan(build, root, desc, **custom_vars): vars["build"] = build vars.update(custom_vars) - arg = vars, debug, descriptions - os.path.walk(root, walk, arg) + numStamped = 0 + for directory, dirnames, filenames in os.walk(root): + numStamped += walk(vars, debug, descriptions, directory, filenames) - print("Stamped %d files." % (numStamped)) + print(f"Stamped {numStamped} files.") if __name__ == "__main__": @@ -146,4 +153,4 @@ def scan(build, root, desc, **custom_vars): print("ERROR: incorrect invocation. See script's header comments.") sys.exit(1) - scan(*tuple(sys.argv[1:])) + scan(*sys.argv[1:])