forked from pypa/setuptools
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CVE-2024-6345 Cherry Pick fix 88807c7
Merge changes from: pypa@88807c7 Then fixup problems.
- Loading branch information
Showing
5 changed files
with
141 additions
and
130 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Modernized and refactored VCS handling in package_index. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,45 +1,33 @@ | ||
| """PyPI and direct package downloading.""" | ||
|
|
||
| import sys | ||
| import os | ||
| import re | ||
| import io | ||
| import shutil | ||
| import socket | ||
| import base64 | ||
| import hashlib | ||
| import itertools | ||
| import configparser | ||
| import hashlib | ||
| import html | ||
| import http.client | ||
| import io | ||
| import itertools | ||
| import os | ||
| import re | ||
| import shutil | ||
| import socket | ||
| import subprocess | ||
| import sys | ||
| import urllib.error | ||
| import urllib.parse | ||
| import urllib.request | ||
| import urllib.error | ||
| from functools import wraps | ||
|
|
||
| import setuptools | ||
| from pkg_resources import ( | ||
| CHECKOUT_DIST, | ||
| Distribution, | ||
| BINARY_DIST, | ||
| normalize_path, | ||
| SOURCE_DIST, | ||
| Environment, | ||
| find_distributions, | ||
| safe_name, | ||
| safe_version, | ||
| to_filename, | ||
| Requirement, | ||
| DEVELOP_DIST, | ||
| EGG_DIST, | ||
| parse_version, | ||
| ) | ||
| from distutils import log | ||
| from distutils.errors import DistutilsError | ||
| from fnmatch import translate | ||
| from setuptools.wheel import Wheel | ||
| from setuptools.extern.more_itertools import unique_everseen | ||
| from functools import wraps | ||
|
|
||
| import setuptools | ||
| from pkg_resources import (BINARY_DIST, CHECKOUT_DIST, DEVELOP_DIST, EGG_DIST, | ||
| SOURCE_DIST, Distribution, Environment, Requirement, | ||
| find_distributions, normalize_path, parse_version, safe_name, | ||
| safe_version, to_filename) | ||
| from setuptools.extern.more_itertools import unique_everseen | ||
| from setuptools.wheel import Wheel | ||
|
|
||
| EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') | ||
| HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) | ||
|
|
@@ -195,7 +183,7 @@ def interpret_distro_name( | |
| '-'.join(parts[p:]), | ||
| py_version=py_version, | ||
| precedence=precedence, | ||
| platform=platform | ||
| platform=platform, | ||
| ) | ||
|
|
||
|
|
||
|
|
@@ -305,7 +293,7 @@ def __init__( | |
| ca_bundle=None, | ||
| verify_ssl=True, | ||
| *args, | ||
| **kw | ||
| **kw, | ||
| ): | ||
| super().__init__(*args, **kw) | ||
| self.index_url = index_url + "/"[: not index_url.endswith('/')] | ||
|
|
@@ -586,7 +574,7 @@ def download(self, spec, tmpdir): | |
| scheme = URL_SCHEME(spec) | ||
| if scheme: | ||
| # It's a url, download it to tmpdir | ||
| found = self._download_url(scheme.group(1), spec, tmpdir) | ||
| found = self._download_url(spec, tmpdir) | ||
| base, fragment = egg_info_for_url(spec) | ||
| if base.endswith('.py'): | ||
| found = self.gen_setup(found, fragment, tmpdir) | ||
|
|
@@ -813,7 +801,7 @@ def open_url(self, url, warning=None): # noqa: C901 # is too complex (12) | |
| else: | ||
| raise DistutilsError("Download error for %s: %s" % (url, v)) from v | ||
|
|
||
| def _download_url(self, scheme, url, tmpdir): | ||
| def _download_url(self, url, tmpdir): | ||
| # Determine download filename | ||
| # | ||
| name, fragment = egg_info_for_url(url) | ||
|
|
@@ -828,19 +816,59 @@ def _download_url(self, scheme, url, tmpdir): | |
|
|
||
| filename = os.path.join(tmpdir, name) | ||
|
|
||
| # Download the file | ||
| # | ||
| if scheme == 'svn' or scheme.startswith('svn+'): | ||
| return self._download_svn(url, filename) | ||
| elif scheme == 'git' or scheme.startswith('git+'): | ||
| return self._download_git(url, filename) | ||
| elif scheme.startswith('hg+'): | ||
| return self._download_hg(url, filename) | ||
| elif scheme == 'file': | ||
| return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) | ||
| else: | ||
| self.url_ok(url, True) # raises error if not allowed | ||
| return self._attempt_download(url, filename) | ||
| return self._download_vcs(url, filename) or self._download_other(url, filename) | ||
|
|
||
| @staticmethod | ||
| def _resolve_vcs(url): | ||
| """ | ||
| >>> rvcs = PackageIndex._resolve_vcs | ||
| >>> rvcs('git+http://foo/bar') | ||
| 'git' | ||
| >>> rvcs('hg+https://foo/bar') | ||
| 'hg' | ||
| >>> rvcs('git:myhost') | ||
| 'git' | ||
| >>> rvcs('hg:myhost') | ||
| >>> rvcs('http://foo/bar') | ||
| """ | ||
| scheme = urllib.parse.urlsplit(url).scheme | ||
| pre, sep, post = scheme.partition('+') | ||
| # svn and git have their own protocol; hg does not | ||
| allowed = set(['svn', 'git'] + ['hg'] * bool(sep)) | ||
| return next(iter({pre} & allowed), None) | ||
|
|
||
| def _download_vcs(self, url, spec_filename): | ||
| vcs = self._resolve_vcs(url) | ||
| if not vcs: | ||
| return | ||
| if vcs == 'svn': | ||
| raise DistutilsError( | ||
| f"Invalid config, SVN download is not supported: {url}" | ||
| ) | ||
|
|
||
| filename, _, _ = spec_filename.partition('#') | ||
| url, rev = self._vcs_split_rev_from_url(url) | ||
|
|
||
| self.info(f"Doing {vcs} clone from {url} to {filename}") | ||
| subprocess.check_call([vcs, 'clone', '--quiet', url, filename]) | ||
|
|
||
| co_commands = dict( | ||
| git=[vcs, '-C', filename, 'checkout', '--quiet', rev], | ||
| hg=[vcs, '--cwd', filename, 'up', '-C', '-r', rev, '-q'], | ||
| ) | ||
| if rev is not None: | ||
| self.info(f"Checking out {rev}") | ||
| subprocess.check_call(co_commands[vcs]) | ||
|
|
||
| return filename | ||
|
|
||
| def _download_other(self, url, filename): | ||
| scheme = urllib.parse.urlsplit(url).scheme | ||
| if scheme == 'file': # pragma: no cover | ||
| return urllib.request.url2pathname(urllib.parse.urlparse(url).path) | ||
| # raise error if not allowed | ||
| self.url_ok(url, True) | ||
| return self._attempt_download(url, filename) | ||
|
|
||
| def scan_url(self, url): | ||
| self.process_url(url, True) | ||
|
|
@@ -856,64 +884,37 @@ def _invalid_download_html(self, url, headers, filename): | |
| os.unlink(filename) | ||
| raise DistutilsError(f"Unexpected HTML page found at {url}") | ||
|
|
||
| def _download_svn(self, url, _filename): | ||
| raise DistutilsError(f"Invalid config, SVN download is not supported: {url}") | ||
|
|
||
| @staticmethod | ||
| def _vcs_split_rev_from_url(url, pop_prefix=False): | ||
| scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) | ||
| def _vcs_split_rev_from_url(url): | ||
| """ | ||
| Given a possible VCS URL, return a clean URL and resolved revision if any. | ||
| >>> vsrfu = PackageIndex._vcs_split_rev_from_url | ||
| >>> vsrfu('git+https://github.com/pypa/[email protected]#egg-info=setuptools') | ||
| ('https://github.com/pypa/setuptools', 'v69.0.0') | ||
| >>> vsrfu('git+https://github.com/pypa/setuptools#egg-info=setuptools') | ||
| ('https://github.com/pypa/setuptools', None) | ||
| >>> vsrfu('http://foo/bar') | ||
| ('http://foo/bar', None) | ||
| """ | ||
| parts = urllib.parse.urlsplit(url) | ||
|
|
||
| scheme = scheme.split('+', 1)[-1] | ||
| clean_scheme = parts.scheme.split('+', 1)[-1] | ||
|
|
||
| # Some fragment identification fails | ||
| path = path.split('#', 1)[0] | ||
| no_fragment_path, _, _ = parts.path.partition('#') | ||
|
|
||
| rev = None | ||
| if '@' in path: | ||
| path, rev = path.rsplit('@', 1) | ||
| pre, sep, post = no_fragment_path.rpartition('@') | ||
| clean_path, rev = (pre, post) if sep else (post, None) | ||
|
|
||
| # Also, discard fragment | ||
| url = urllib.parse.urlunsplit((scheme, netloc, path, query, '')) | ||
| resolved = parts._replace( | ||
| scheme=clean_scheme, | ||
| path=clean_path, | ||
| # discard the fragment | ||
| fragment='', | ||
| ).geturl() | ||
|
|
||
| return url, rev | ||
|
|
||
| def _download_git(self, url, filename): | ||
| filename = filename.split('#', 1)[0] | ||
| url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) | ||
|
|
||
| self.info("Doing git clone from %s to %s", url, filename) | ||
| os.system("git clone --quiet %s %s" % (url, filename)) | ||
|
|
||
| if rev is not None: | ||
| self.info("Checking out %s", rev) | ||
| os.system( | ||
| "git -C %s checkout --quiet %s" | ||
| % ( | ||
| filename, | ||
| rev, | ||
| ) | ||
| ) | ||
|
|
||
| return filename | ||
|
|
||
| def _download_hg(self, url, filename): | ||
| filename = filename.split('#', 1)[0] | ||
| url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) | ||
|
|
||
| self.info("Doing hg clone from %s to %s", url, filename) | ||
| os.system("hg clone --quiet %s %s" % (url, filename)) | ||
|
|
||
| if rev is not None: | ||
| self.info("Updating to %s", rev) | ||
| os.system( | ||
| "hg --cwd %s up -C -r %s -q" | ||
| % ( | ||
| filename, | ||
| rev, | ||
| ) | ||
| ) | ||
|
|
||
| return filename | ||
| return resolved, rev | ||
|
|
||
| def debug(self, msg, *args): | ||
| log.debug(msg, *args) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters