From 590900aa35ef44f96b9c50fd7da907a0058514cf Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 19 Jun 2024 13:41:29 +0200 Subject: [PATCH] Add support for editable pip installs (#36) * Add support for editable pip installs * fix early return * add test --- .gitignore | 2 ++ conda_pypi/cli/pip.py | 27 +++++++++++++++-- conda_pypi/dependencies/__init__.py | 18 ++++++++--- conda_pypi/dependencies/pip.py | 18 ++++++++--- conda_pypi/main.py | 4 +-- tests/test_install.py | 47 ++++++++++++++++++++++++++++- 6 files changed, 101 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index d2a910d..7b4ec98 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,8 @@ dmypy.json # Used in debugging explicit.txt +# pip editable clones stuff here +src/ # hatch-vcs conda_pypi/_version.py diff --git a/conda_pypi/cli/pip.py b/conda_pypi/cli/pip.py index 507e203..313bdc9 100644 --- a/conda_pypi/cli/pip.py +++ b/conda_pypi/cli/pip.py @@ -15,6 +15,7 @@ add_parser_help, add_parser_prefix, ) +from conda.exceptions import ArgumentError logger = getLogger(f"conda.{__name__}") @@ -56,6 +57,12 @@ def configure_parser(parser: argparse.ArgumentParser): default="conda-forge", help="Where to look for conda dependencies.", ) + install.add_argument( + "-e", "--editable", + metavar="", + help="Install a project in editable mode (i.e. setuptools 'develop mode') " + "from a local project path or a VCS url." + ) install.add_argument( "--backend", metavar="TOOL", @@ -63,10 +70,20 @@ def configure_parser(parser: argparse.ArgumentParser): choices=BACKENDS, help="Which tool to use for PyPI packaging dependency resolution.", ) - install.add_argument("packages", metavar="package", nargs="+") + install.add_argument("packages", metavar="package", nargs="*") def execute(args: argparse.Namespace) -> int: + if not args.packages and not args.editable: + raise ArgumentError( + "No packages requested. Please provide one or more packages, " + "or one editable specification." + ) + if args.editable and args.backend == "grayskull": + raise ArgumentError( + "--editable PKG and --backend=grayskull are not compatible. Please use --backend=pip." + ) + from conda.common.io import Spinner from conda.models.match_spec import MatchSpec from ..dependencies import analyze_dependencies @@ -82,13 +99,14 @@ def execute(args: argparse.Namespace) -> int: packages_not_installed = validate_target_env(prefix, args.packages) packages_to_process = args.packages if args.force_reinstall else packages_not_installed - if not packages_to_process: + if not packages_to_process and not args.editable: print("All packages are already installed.", file=sys.stderr) return 0 with Spinner("Analyzing dependencies", enabled=not args.quiet, json=args.json): - conda_deps, pypi_deps = analyze_dependencies( + conda_deps, pypi_deps, editable_deps = analyze_dependencies( *packages_to_process, + editable=args.editable, prefer_on_conda=not args.force_with_pip, channel=args.conda_channel, backend=args.backend, @@ -113,6 +131,9 @@ def execute(args: argparse.Namespace) -> int: logger.warning("ignoring extra specifiers for %s: %s", name, specs[1:]) spec = spec.replace(" ", "") # remove spaces pypi_specs.append(spec) + for name, specs in editable_deps.items(): + for spec in specs: + pypi_specs.append(f"--editable={spec}") if not args.quiet or not args.json: if conda_match_specs: diff --git a/conda_pypi/dependencies/__init__.py b/conda_pypi/dependencies/__init__.py index d0a43ed..3a67ff8 100644 --- a/conda_pypi/dependencies/__init__.py +++ b/conda_pypi/dependencies/__init__.py @@ -34,12 +34,13 @@ def analyze_dependencies( *pypi_specs: str, + editable: str | None = None, prefer_on_conda: bool = True, channel: str = "conda-forge", backend: Literal["grayskull", "pip"] = "pip", prefix: str | os.PathLike | None = None, force_reinstall: bool = False, -) -> tuple[dict[str, list[str]], dict[str, list[str]]]: +) -> tuple[dict[str, list[str]], dict[str, list[str]], dict[str, list[str]]]: conda_deps = defaultdict(list) needs_analysis = [] for pypi_spec in pypi_specs: @@ -56,11 +57,15 @@ def analyze_dependencies( conda_deps[MatchSpec(conda_spec).name].append(conda_spec) continue needs_analysis.append(pypi_spec) + if editable: + needs_analysis.extend(["-e", editable]) if not needs_analysis: - return conda_deps, {} + return conda_deps, {}, {} if backend == "grayskull": + if editable: + logger.warning("Ignoring editable=%s with backend=grayskull", editable) from .grayskull import _analyze_with_grayskull found_conda_deps, pypi_deps = _analyze_with_grayskull( @@ -69,16 +74,18 @@ def analyze_dependencies( elif backend == "pip": from .pip import _analyze_with_pip - python_deps, pypi_deps = _analyze_with_pip( + python_deps, pypi_deps, editable_deps = _analyze_with_pip( *needs_analysis, prefix=prefix, force_reinstall=force_reinstall, - ) + ) + found_conda_deps, pypi_deps = _classify_dependencies( pypi_deps, prefer_on_conda=prefer_on_conda, channel=channel, ) + found_conda_deps.update(python_deps) else: raise ValueError(f"Unknown backend {backend}") @@ -89,7 +96,8 @@ def analyze_dependencies( # deduplicate conda_deps = {name: list(dict.fromkeys(specs)) for name, specs in conda_deps.items()} pypi_deps = {name: list(dict.fromkeys(specs)) for name, specs in pypi_deps.items()} - return conda_deps, pypi_deps + editable_deps = editable_deps if editable else {} + return conda_deps, pypi_deps, editable_deps def _classify_dependencies( diff --git a/conda_pypi/dependencies/pip.py b/conda_pypi/dependencies/pip.py index fbd1c9b..003ee0e 100644 --- a/conda_pypi/dependencies/pip.py +++ b/conda_pypi/dependencies/pip.py @@ -12,17 +12,27 @@ def _analyze_with_pip( *packages: str, prefix: str | None = None, force_reinstall: bool = False, -) -> tuple[dict[str, list[str]], dict[str, list[str]]]: +) -> tuple[dict[str, list[str]], dict[str, list[str]], dict[str, list[str]]]: report = dry_run_pip_json(("--prefix", prefix, *packages), force_reinstall) deps_from_pip = defaultdict(list) + editable_deps = defaultdict(list) conda_deps = defaultdict(list) for item in report["install"]: metadata = item["metadata"] logger.debug("Analyzing %s", metadata["name"]) logger.debug(" metadata: %s", json.dumps(metadata, indent=2)) - deps_from_pip[metadata["name"]].append(f"{metadata['name']}=={metadata['version']}") + if item.get("download_info", {}).get("dir_info", {}).get("editable"): + editable_deps[metadata["name"]].append(item["download_info"]["url"]) + elif item.get("is_direct"): + deps_from_pip[metadata["name"]].append(item["download_info"]["url"]) + else: + deps_from_pip[metadata["name"]].append(f"{metadata['name']}=={metadata['version']}") if python_version := metadata.get("requires_python"): conda_deps["python"].append(f"python {python_version}") - deps_from_pip = {name: list(dict.fromkeys(specs)) for name, specs in deps_from_pip.items()} - return conda_deps, deps_from_pip + deps_from_pip = { + name: list(dict.fromkeys(specs)) + for name, specs in deps_from_pip.items() + if name not in editable_deps + } + return conda_deps, deps_from_pip, editable_deps diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 2aae29b..768dc06 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -139,7 +139,7 @@ def run_pip_install( if check and process.returncode: raise CondaError( f"Failed to run pip:\n" - f" command: {shlex.join(command)}\n" + f" command: {shlex.join(map(str,command))}\n" f" exit code: {process.returncode}\n" f" stderr:\n{process.stderr}\n" f" stdout:\n{process.stdout}" @@ -265,7 +265,7 @@ def dry_run_pip_json( if process.returncode != 0: raise CondaError( f"Failed to dry-run pip:\n" - f" command: {shlex.join(cmd)}\n" + f" command: {shlex.join(map(str, cmd))}\n" f" exit code: {process.returncode}\n" f" stderr:\n{process.stderr}\n" f" stdout:\n{process.stdout}" diff --git a/tests/test_install.py b/tests/test_install.py index e60cfce..682cc46 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import sys from pathlib import Path from subprocess import run @@ -11,7 +12,7 @@ from conda.testing import CondaCLIFixture, TmpEnvFixture from conda_pypi.dependencies import NAME_MAPPINGS, BACKENDS, _pypi_spec_to_conda_spec -from conda_pypi.python_paths import get_env_python +from conda_pypi.python_paths import get_env_python, get_env_site_packages @pytest.mark.parametrize("source", NAME_MAPPINGS.keys()) @@ -194,3 +195,47 @@ def test_lockfile_roundtrip( print(err2, file=sys.stderr) assert rc2 == 0 assert sorted(out2.splitlines()) == sorted(out.splitlines()) + + +@pytest.mark.parametrize( + "requirement,name", + [ + ( + # pure Python + "git+https://github.com/dateutil/dateutil.git@2.9.0.post0", + "python_dateutil", + ), + ( + # compiled bits + "git+https://github.com/yaml/pyyaml.git@6.0.1", + "PyYAML", + ), + ( + # has conda dependencies + "git+https://github.com/regro/conda-forge-metadata.git@0.8.1", + "conda_forge_metadata", + ), + ], +) +def test_editable_installs( + tmp_path: Path, tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture, requirement, name +): + os.chdir(tmp_path) + with tmp_env("python=3.9", "pip") as prefix: + out, err, rc = conda_cli( + "pip", + "-p", + prefix, + "--yes", + "install", + "-e", + f"{requirement}#egg={name}", + ) + print(out) + print(err, file=sys.stderr) + assert rc == 0 + sp = get_env_site_packages(prefix) + editable_pth = list(sp.glob(f"__editable__.{name}-*.pth")) + assert len(editable_pth) == 1 + pth_contents = editable_pth[0].read_text().strip() + assert pth_contents.startswith((str(tmp_path / "src"), f"import __editable___{name}"))