Switch to setuptools for the Pex build. (#2599)

As noted in a TODO, now removed, and re-emphasized by a recent break as
Hatchling moved to support PEP-639, it was time to move off Hatchling to
a simpler build system that supported all the Python versions Pex
supports.

MANIFEST.in

@@ -0,0 +1,10 @@
+include CHANGES.md docs-requirements.txt dtox.sh mypy.ini tox.ini
+graft assets
+graft build-backend
+graft docker
+graft docs
+graft package
+graft scripts
+graft testing
+graft tests
+global-exclude *.pyc *.pyo

build-backend/pex_build/__init__.py

@@ -1,6 +1,8 @@
 # Copyright 2024 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from __future__ import absolute_import
+
 import os
 
 # When running under MyPy, this will be set to True for us automatically; so we can use it as a

build-backend/pex_build/setuptools/build.py

@@ -0,0 +1,97 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import hashlib
+import os.path
+import subprocess
+import sys
+
+import pex_build
+import setuptools.build_meta
+
+# We re-export all setuptools' PEP-517 build backend hooks here for the build frontend to call.
+from setuptools.build_meta import *  # NOQA
+
+from pex import hashing
+from pex.common import open_zip, temporary_dir
+from pex.pep_376 import Hash, InstalledFile, Record
+from pex.version import __version__
+
+if pex_build.TYPE_CHECKING:
+    from typing import Any, Dict, List, Optional
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    # type: (Optional[Dict[str, Any]]) -> List[str]
+
+    reqs = setuptools.build_meta.get_requires_for_build_wheel(
+        config_settings=config_settings
+    )  # type: List[str]
+    if pex_build.INCLUDE_DOCS:
+        with open("docs-requirements.txt") as fp:
+            for raw_req in fp.readlines():
+                req = raw_req.strip()
+                if not req or req.startswith("#"):
+                    continue
+                reqs.append(req)
+    return reqs
+
+
+def build_wheel(
+    wheel_directory,  # type: str
+    config_settings=None,  # type: Optional[Dict[str, Any]]
+    metadata_directory=None,  # type: Optional[str]
+):
+    # type: (...) -> str
+
+    wheel = setuptools.build_meta.build_wheel(
+        wheel_directory, config_settings=config_settings, metadata_directory=metadata_directory
+    )  # type: str
+    if pex_build.INCLUDE_DOCS:
+        wheel_path = os.path.join(wheel_directory, wheel)
+        with temporary_dir() as chroot:
+            with open_zip(wheel_path) as zip_fp:
+                zip_fp.extractall(chroot)
+
+            out_dir = os.path.join(chroot, "pex", "docs")
+            subprocess.check_call(
+                args=[
+                    sys.executable,
+                    os.path.join("scripts", "build-docs.py"),
+                    "--clean-html",
+                    out_dir,
+                ]
+            )
+            dist_info_dir = "pex-{version}.dist-info".format(version=__version__)
+            record_path = os.path.join(chroot, dist_info_dir, "RECORD")
+            with open(record_path) as fp:
+                installed_files = list(Record.read(fp))
+            for root, _, files in os.walk(out_dir):
+                for f in files:
+                    src = os.path.join(root, f)
+                    dst = os.path.relpath(src, chroot)
+                    hasher = hashlib.sha256()
+                    hashing.file_hash(src, digest=hasher)
+                    installed_files.append(
+                        InstalledFile(path=dst, hash=Hash.create(hasher), size=os.path.getsize(src))
+                    )
+            Record.write(record_path, installed_files)
+            with open_zip(wheel_path, "w") as zip_fp:
+
+                def add_top_level_dir(name):
+                    # type: (str) -> None
+                    top = os.path.join(chroot, name)
+                    zip_fp.write(top, name + "/")
+                    for root, dirs, files in os.walk(top):
+                        dirs[:] = sorted(dirs)
+                        for path in sorted(files) + dirs:
+                            src = os.path.join(root, path)
+                            dst = os.path.relpath(src, chroot)
+                            zip_fp.write(src, dst)
+
+                add_top_level_dir("pex")
+                add_top_level_dir(dist_info_dir)
+
+    return wheel

mypy.ini

@@ -49,9 +49,5 @@ ignore_missing_imports = True
 [mypy-pkg_resources]
 ignore_missing_imports = True
 
-[mypy-hatchling.*]
-# Currently we use one venv for MyPy checks and the hatchling requirements are not compatible with
-# the rest; so we ignore. Ideally pex.hatchling build backend type-checking would be segregated as
-# scripts are today and each with its own venv.
+[mypy-setuptools.*]
 ignore_missing_imports = True
-

pex/vendor/__init__.py

@@ -151,7 +151,14 @@ def create_packages(self):
         if not self.rewrite:
             # The extra package structure is only required by Pex for vendored code used via import
             # rewrites.
-            return
+
+            # N.B.: Although we've historically early-returned here, the switch from flit to
+            # setuptools for our build backend necessitates all vendored dists are seen as part of
+            # the `pex` package tree by setuptools to get all vendored code properly included in
+            # our distribution.
+            # TODO(John Sirois): re-introduce early return once it is no longer foils our build
+            #  process.
+            pass
 
         for index, _ in enumerate(self._subpath_components):
             relpath = _PACKAGE_COMPONENTS + self._subpath_components[: index + 1] + ["__init__.py"]

pex/vendor/_vendored/pip/.layout.json

@@ -1 +1 @@
-{"fingerprint": "dae4c964a32e77aac0b92730a1722943ce1ff786660e0175952d3402f79e8fa4", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"}
+{"fingerprint": "6886e39e75a6d2abaa5ed786b8e5e135b533e338316f78ace921ffaa46c553cf", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"}