Merge pull request #563 from pdm-project/new-installer

Author: frostming

news/519.refactor.md

@@ -0,0 +1 @@
+Use installer as the wheel installer, replacing `distlib`.

news/529.bugfix.md

@@ -0,0 +1 @@
+Accept non-canonical distribution name in the wheel's dist-info directory name.

pdm.lock

@@ -513,7 +513,7 @@ def version_matcher(py_version: PythonInfo) -> bool:
         and not project.environment.is_global
     ):
         project.core.ui.echo(termui.cyan("Updating executable scripts..."))
-        project.environment.update_shebangs(new_path)
+        project.environment.update_shebangs(old_path, new_path)
 
 
 def do_import(

pdm/cli/actions.py

@@ -1,15 +1,17 @@
 from __future__ import annotations
 
 import pathlib
+import sys
 from typing import TYPE_CHECKING
 
-import distlib.scripts
-from distlib.wheel import Wheel
+from installer.exceptions import InvalidWheelSource
+from installer.sources import WheelFile as _WheelFile
 from pip._vendor.pkg_resources import EggInfoDistribution
 
 from pdm import termui
 from pdm.models import pip_shims
 from pdm.models.requirements import parse_requirement
+from pdm.utils import cached_property
 
 if TYPE_CHECKING:
     from pip._vendor.pkg_resources import Distribution
@@ -30,6 +32,23 @@ def format_dist(dist: Distribution) -> str:
     return formatter.format(version=termui.yellow(dist.version), path=path)
 
 
+class WheelFile(_WheelFile):
+    @cached_property
+    def dist_info_dir(self) -> str:
+        namelist = self._zipfile.namelist()
+        try:
+            return next(
+                name.split("/")[0]
+                for name in namelist
+                if name.split("/")[0].endswith(".dist-info")
+            )
+        except StopIteration:  # pragma: no cover
+            canonical_name = super().dist_info_dir
+            raise InvalidWheelSource(
+                f"The wheel doesn't contain metadata {canonical_name!r}"
+            )
+
+
 class Installer:  # pragma: no cover
     """The installer that performs the installation and uninstallation actions."""
 
@@ -44,17 +63,27 @@ def install(self, candidate: Candidate) -> None:
             self.install_editable(candidate.ireq)
         else:
             built = candidate.build()
-            self.install_wheel(Wheel(built))
+            self.install_wheel(built)
 
-    def install_wheel(self, wheel: Wheel) -> None:
-        paths = self.environment.get_paths()
-        maker = distlib.scripts.ScriptMaker(None, None)
-        maker.variants = set(("",))
-        enquoted_executable = distlib.scripts.enquote_executable(
-            self.environment.interpreter.executable
+    def install_wheel(self, wheel: str) -> None:
+        from installer import __version__, install
+        from installer.destinations import SchemeDictionaryDestination
+
+        destination = SchemeDictionaryDestination(
+            self.environment.get_paths(),
+            interpreter=self.environment.interpreter.executable,
+            script_kind=self._get_kind(),
         )
-        maker.executable = enquoted_executable
-        wheel.install(paths, maker)
+
+        with WheelFile.open(wheel) as source:
+            install(
+                source=source,
+                destination=destination,
+                # Additional metadata that is generated by the installation tool.
+                additional_metadata={
+                    "INSTALLER": f"installer {__version__}".encode(),
+                },
+            )
 
     def install_editable(self, ireq: pip_shims.InstallRequirement) -> None:
         from pdm.builders import EditableBuilder
@@ -83,3 +112,13 @@ def uninstall(self, dist: Distribution) -> None:
         pathset = ireq.uninstall(auto_confirm=self.auto_confirm)
         if pathset:
             pathset.commit()
+
+    def _get_kind(self) -> str:
+        if sys.platform != "win32":
+            return "posix"
+        is_32bit = self.environment.interpreter.is_32bit
+        # TODO: support win arm64
+        if is_32bit:
+            return "win-ia32"
+        else:
+            return "win-amd64"

pdm/installers/installers.py

@@ -2,7 +2,7 @@
 
 import collections
 import os
-import re
+import shlex
 import shutil
 import subprocess
 import sys
@@ -12,7 +12,6 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Iterator
 
-from distlib.scripts import ScriptMaker
 from pip._vendor import packaging, pkg_resources
 
 from pdm import termui
@@ -33,6 +32,19 @@
     from pdm.project import Project
 
 
+def _get_shebang_path(executable: str, is_launcher: bool) -> bytes:
+    """Get the interpreter path in the shebang line
+
+    The launcher can just use the command as-is.
+    Otherwise if the path contains whitespace or is too long, both distlib
+    and installer use a clever hack to make the shebang after ``/bin/sh``,
+    where the interpreter path is quoted.
+    """
+    if is_launcher or " " not in executable and (len(executable) + 3) <= 127:
+        return executable.encode("utf-8")
+    return shlex.quote(executable).encode("utf-8")
+
+
 class WorkingSet(collections.abc.Mapping):
     """A dict-like class that holds all installed packages in the lib directory."""
 
@@ -217,18 +229,16 @@ def which(self, command: str) -> str | None:
         new_path = os.pathsep.join([python_root, this_path, os.getenv("PATH", "")])
         return shutil.which(command, path=new_path)
 
-    def update_shebangs(self, new_path: str) -> None:
+    def update_shebangs(self, old_path: str, new_path: str) -> None:
         """Update the shebang lines"""
         scripts = self.get_paths()["scripts"]
-        maker = ScriptMaker(None, None)
-        maker.executable = new_path
-        shebang = maker._get_shebang("utf-8").rstrip().replace(b"\\", b"\\\\")
         for child in Path(scripts).iterdir():
             if not child.is_file() or child.suffix not in (".exe", ".py", ""):
                 continue
-            child.write_bytes(
-                re.sub(rb"#!.+?python.*?$", shebang, child.read_bytes(), flags=re.M)
-            )
+            is_launcher = child.suffix == ".exe"
+            old_shebang = _get_shebang_path(old_path, is_launcher)
+            new_shebang = _get_shebang_path(new_path, is_launcher)
+            child.write_bytes(child.read_bytes().replace(old_shebang, new_shebang, 1))
 
     def _download_pip_wheel(self, path: str | Path) -> None:
         dirname = Path(tempfile.mkdtemp(prefix="pip-download-"))

pdm/models/environment.py

@@ -12,7 +12,6 @@ dependencies = [
     "appdirs",
     "atoml>=1.0.3",
     "click>=7",
-    "distlib>=0.3.1",
     "importlib-metadata; python_version < \"3.8\"",
     "pdm-pep517>=0.8,<0.9",
     "pep517",
@@ -23,6 +22,7 @@ dependencies = [
     "shellingham<2.0.0,>=1.3.2",
     "wheel<1.0.0,>=0.36.2",
     "tomli~=1.0",
+    "installer~=0.2",
 ]
 name = "pdm"
 description = "Python Development Master"

pyproject.toml

@@ -2,7 +2,6 @@
 import sys
 
 import pytest
-from distlib.wheel import Wheel
 
 from pdm.cli import actions
 from pdm.exceptions import InvalidPyVersion, PdmException, PdmUsageError
@@ -222,8 +221,8 @@ def test_lock_dependencies(project):
 def test_build_distributions(tmp_path, core):
     project = core.create_project()
     actions.do_build(project, dest=tmp_path.as_posix())
-    wheel = Wheel(next(tmp_path.glob("*.whl")).as_posix())
-    assert wheel.name == "pdm"
+    wheel = next(tmp_path.glob("*.whl"))
+    assert wheel.name.startswith("pdm-")
     tarball = next(tmp_path.glob("*.tar.gz"))
     assert tarball.exists()
 

tests/cli/test_actions.py

@@ -1,8 +1,6 @@
 import tarfile
 import zipfile
 
-from distlib.wheel import Wheel
-
 from pdm.cli import actions
 
 
@@ -40,10 +38,6 @@ def test_build_single_module(fixture_project):
     for name in ("pyproject.toml", "LICENSE"):
         assert name not in zip_names
 
-    assert Wheel(
-        (project.root / "dist/demo_module-0.1.0-py3-none-any.whl").as_posix()
-    ).metadata
-
 
 def test_build_single_module_with_readme(fixture_project):
     project = fixture_project("demo-module")

tests/cli/test_build.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50f11ea3c386c4330c7a8be92c0736b4b6a5924bdbd00b8b93f93b0c7fe6f2b3
+size 49497

tests/fixtures/artifacts/PyFunctional-1.4.3-py3-none-any.whl

@@ -0,0 +1,16 @@
+from pdm.installers import Installer
+from pdm.models.candidates import Candidate
+from pdm.models.pip_shims import Link
+from pdm.models.requirements import parse_requirement
+
+
+def test_install_wheel_with_inconsistent_dist_info(project):
+    req = parse_requirement("pyfunctional")
+    candidate = Candidate(
+        req,
+        project.environment,
+        link=Link("http://fixtures.test/artifacts/PyFunctional-1.4.3-py3-none-any.whl"),
+    )
+    installer = Installer(project.environment)
+    installer.install(candidate)
+    assert "pyfunctional" in project.environment.get_working_set()

tests/test_installer.py

@@ -3,11 +3,8 @@
 import venv
 from pathlib import Path
 
-import distlib.wheel
 import pytest
 
-from pdm.models.requirements import filter_requirements_with_extras
-from pdm.pep517.api import build_wheel
 from pdm.utils import cd, temp_environ
 
 
@@ -92,20 +89,6 @@ def test_project_use_venv(project):
     assert env.is_global
 
 
-def test_project_with_combined_extras(fixture_project):
-    project = fixture_project("demo-combined-extras")
-    (project.root / "build").mkdir(exist_ok=True)
-    with cd(project.root.as_posix()):
-        wheel_name = build_wheel(str(project.root / "build"))
-        wheel = distlib.wheel.Wheel(str(project.root / "build" / wheel_name))
-
-    all_requires = filter_requirements_with_extras(
-        wheel.metadata.run_requires, ("all",)
-    )
-    for dep in ("urllib3", "chardet", "idna"):
-        assert dep in all_requires
-
-
 def test_project_packages_path(project):
     packages_path = project.environment.packages_path
     version = ".".join(map(str, sys.version_info[:2]))