diff --git a/changelog/1180.misc.txt b/changelog/1180.misc.txt new file mode 100644 index 00000000..9e00151b --- /dev/null +++ b/changelog/1180.misc.txt @@ -0,0 +1,8 @@ +- ``packaging`` is used instead of ``pkginfo`` for parsing and + validating metadata. This aligns metadata validation to the one + performed by PyPI. ``pkginfo`` is not a dependency anymore. +- Metadata fields defined added with metadata version 2.4 as defined + by PEP 639 are now sent to the package index when a distribution is + uploaded. This results in licensing information to appear correctly + on the package page on PyPI when uploading packages using metadata + version 2.4. diff --git a/docs/conf.py b/docs/conf.py index ddef188a..e07c70ee 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -283,6 +283,7 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "requests": ("https://requests.readthedocs.io/en/latest/", None), + "packaging": ("https://packaging.pypa.io/en/latest/", None), } # Be strict about the invalid references: @@ -291,7 +292,6 @@ # TODO: Try to add these to intersphinx_mapping nitpick_ignore_regex = [ (r"py:.*", r"pkginfo.*"), - ("py:class", r"warnings\.WarningMessage"), ] # -- Options for apidoc output ------------------------------------------------ diff --git a/pyproject.toml b/pyproject.toml index 582ccaa9..91d7d483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "pkginfo >= 1.8.1", "readme-renderer >= 35.0", "requests >= 2.20", "requests-toolbelt >= 0.8.0, != 0.9.0", @@ -41,7 +40,7 @@ dependencies = [ "keyring >= 15.1; platform_machine != 'ppc64le' and platform_machine != 's390x'", "rfc3986 >= 1.4.0", "rich >= 12.0.0", - "packaging", + "packaging >= 24.0", ] dynamic = ["version"] diff --git a/tests/fixtures/everything.metadata b/tests/fixtures/everything.metadata new file mode 100644 index 00000000..eea1351b --- /dev/null +++ b/tests/fixtures/everything.metadata @@ -0,0 +1,44 @@ +Metadata-Version: 2.4 +Name: BeagleVote +Version: 1.0a2 +Platform: ObscureUnix +Platform: RareDOS +Supported-Platform: RedHat 7.2 +Supported-Platform: i386-win32-2791 +Summary: A module for collecting votes from beagles. +Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM +Keywords: dog,puppy,voting,election +Home-page: http://www.example.com/~cschultz/bvote/ +Download-URL: …/BeagleVote-0.45.tgz +Author: C. Schultz, Universal Features Syndicate, + Los Angeles, CA +Author-email: "C. Schultz" +Maintainer: C. Schultz, Universal Features Syndicate, + Los Angeles, CA +Maintainer-email: "C. Schultz" +License: This software may only be obtained by sending the + author a postcard, and then the user promises not + to redistribute it. +License-Expression: Apache-2.0 OR BSD-2-Clause +License-File: LICENSE.APACHE +License-File: LICENSE.BSD +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Console (Text Based) +Provides-Extra: pdf +Requires-Dist: reportlab; extra == 'pdf' +Requires-Dist: pkginfo +Requires-Dist: PasteDeploy +Requires-Dist: zope.interface (>3.5.0) +Requires-Dist: pywin32 >1.0; sys_platform == 'win32' +Requires-Python: >=3 +Requires-External: C +Requires-External: libpng (>=1.5) +Requires-External: make; sys_platform != "win32" +Project-URL: Bug Tracker, http://bitbucket.org/tarek/distribute/issues/ +Project-URL: Documentation, https://example.com/BeagleVote +Provides-Dist: OtherProject +Provides-Dist: AnotherProject (3.4) +Provides-Dist: virtual_package; python_version >= "3.4" +Dynamic: Obsoletes-Dist + +This description intentionally left blank. diff --git a/tests/test_package.py b/tests/test_package.py index be3d9540..dfe77a5d 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -31,7 +31,7 @@ def test_sign_file(monkeypatch): package = package_file.PackageFile( filename=filename, comment=None, - metadata=pretend.stub(name="deprecated-pypirc"), + metadata=dict(name="deprecated-pypirc", version="1.2.3"), python_version=None, filetype=None, ) @@ -51,7 +51,7 @@ def test_sign_file_with_identity(monkeypatch): package = package_file.PackageFile( filename=filename, comment=None, - metadata=pretend.stub(name="deprecated-pypirc"), + metadata=dict(name="deprecated-pypirc", version="1.2.3"), python_version=None, filetype=None, ) @@ -106,7 +106,7 @@ def test_package_signed_name_is_correct(): package = package_file.PackageFile( filename=filename, comment=None, - metadata=pretend.stub(name="deprecated-pypirc"), + metadata=dict(name="deprecated-pypirc", version="1.2.3"), python_version=None, filetype=None, ) @@ -163,7 +163,7 @@ def test_package_safe_name_is_correct(pkg_name, expected_name): package = package_file.PackageFile( filename="tests/fixtures/deprecated-pypirc", comment=None, - metadata=pretend.stub(name=pkg_name), + metadata=dict(name=pkg_name, version="1.2.3"), python_version=None, filetype=None, ) @@ -171,10 +171,9 @@ def test_package_safe_name_is_correct(pkg_name, expected_name): assert package.safe_name == expected_name -def test_metadata_dictionary_keys(): - """Merge multiple sources of metadata into a single dictionary.""" - package = package_file.PackageFile.from_filename(helpers.SDIST_FIXTURE, None) - assert set(package.metadata_dictionary()) == { +def test_supported_metadata_fields(): + """Check that ``twine.package.PackageMetadata`` contains the expected fields.""" + assert set(package_file.PackageMetadata.__annotations__) == { # identify release "name", "version", @@ -192,14 +191,16 @@ def test_metadata_dictionary_keys(): "license", "description", "keywords", - "platform", + "platforms", "classifiers", "download_url", - "supported_platform", + "supported_platforms", "comment", "md5_digest", "sha256_digest", "blake2_256_digest", + "gpg_signature", + "attestations", # PEP 314 "provides", "requires", @@ -216,6 +217,9 @@ def test_metadata_dictionary_keys(): "description_content_type", # Metadata 2.2 "dynamic", + # Metadata 2.4 + "license_expression", + "license_files", } @@ -223,9 +227,9 @@ def test_metadata_dictionary_keys(): @pytest.mark.parametrize("attestation", [(None), ({"fake": "attestation"})]) def test_metadata_dictionary_values(gpg_signature, attestation): """Pass values from pkginfo.Distribution through to dictionary.""" - meta = pretend.stub( + meta = dict( name="whatever", - version=pretend.stub(), + version="1.2.3", metadata_version=pretend.stub(), summary=pretend.stub(), home_page=pretend.stub(), @@ -236,10 +240,10 @@ def test_metadata_dictionary_values(gpg_signature, attestation): license=pretend.stub(), description=pretend.stub(), keywords=pretend.stub(), - platforms=pretend.stub(), + platform=pretend.stub(), classifiers=pretend.stub(), download_url=pretend.stub(), - supported_platforms=pretend.stub(), + supported_platform=pretend.stub(), provides=pretend.stub(), requires=pretend.stub(), obsoletes=pretend.stub(), @@ -249,7 +253,7 @@ def test_metadata_dictionary_values(gpg_signature, attestation): requires_dist=pretend.stub(), requires_external=pretend.stub(), requires_python=pretend.stub(), - provides_extras=pretend.stub(), + provides_extra=pretend.stub(), description_content_type=pretend.stub(), dynamic=pretend.stub(), ) @@ -269,48 +273,48 @@ def test_metadata_dictionary_values(gpg_signature, attestation): # identify release assert result["name"] == package.safe_name - assert result["version"] == meta.version + assert result["version"] == package.version == meta["version"] # file content assert result["filetype"] == package.filetype assert result["pyversion"] == package.python_version # additional meta-data - assert result["metadata_version"] == meta.metadata_version - assert result["summary"] == meta.summary - assert result["home_page"] == meta.home_page - assert result["author"] == meta.author - assert result["author_email"] == meta.author_email - assert result["maintainer"] == meta.maintainer - assert result["maintainer_email"] == meta.maintainer_email - assert result["license"] == meta.license - assert result["description"] == meta.description - assert result["keywords"] == meta.keywords - assert result["platform"] == meta.platforms - assert result["classifiers"] == meta.classifiers - assert result["download_url"] == meta.download_url - assert result["supported_platform"] == meta.supported_platforms + assert result["metadata_version"] == meta["metadata_version"] + assert result["summary"] == meta["summary"] + assert result["home_page"] == meta["home_page"] + assert result["author"] == meta["author"] + assert result["author_email"] == meta["author_email"] + assert result["maintainer"] == meta["maintainer"] + assert result["maintainer_email"] == meta["maintainer_email"] + assert result["license"] == meta["license"] + assert result["description"] == meta["description"] + assert result["keywords"] == meta["keywords"] + assert result["platform"] == meta["platform"] + assert result["classifiers"] == meta["classifiers"] + assert result["download_url"] == meta["download_url"] + assert result["supported_platform"] == meta["supported_platform"] assert result["comment"] == package.comment # PEP 314 - assert result["provides"] == meta.provides - assert result["requires"] == meta.requires - assert result["obsoletes"] == meta.obsoletes + assert result["provides"] == meta["provides"] + assert result["requires"] == meta["requires"] + assert result["obsoletes"] == meta["obsoletes"] # Metadata 1.2 - assert result["project_urls"] == meta.project_urls - assert result["provides_dist"] == meta.provides_dist - assert result["obsoletes_dist"] == meta.obsoletes_dist - assert result["requires_dist"] == meta.requires_dist - assert result["requires_external"] == meta.requires_external - assert result["requires_python"] == meta.requires_python + assert result["project_urls"] == meta["project_urls"] + assert result["provides_dist"] == meta["provides_dist"] + assert result["obsoletes_dist"] == meta["obsoletes_dist"] + assert result["requires_dist"] == meta["requires_dist"] + assert result["requires_external"] == meta["requires_external"] + assert result["requires_python"] == meta["requires_python"] # Metadata 2.1 - assert result["provides_extra"] == meta.provides_extras - assert result["description_content_type"] == meta.description_content_type + assert result["provides_extra"] == meta["provides_extra"] + assert result["description_content_type"] == meta["description_content_type"] # Metadata 2.2 - assert result["dynamic"] == meta.dynamic + assert result["dynamic"] == meta["dynamic"] # GPG signature assert result.get("gpg_signature") == gpg_signature @@ -381,41 +385,61 @@ def test_fips_metadata_excludes_md5_and_blake2(monkeypatch): @pytest.mark.parametrize( - "read_data, missing_fields", + "read_data, exception_message", [ pytest.param( - b"Metadata-Version: 2.3\nName: UNKNOWN\nVersion: UNKNOWN\n", - "Name, Version", + b"Metadata-Version: 102.3\nName: test-package\nVersion: 1.0.0\n", + "'102.3' is not a valid metadata version", + id="unsupported Metadata-Version", + ), + pytest.param( + b"Metadata-Version: 2.3\nName: test-package\nVersion: UNKNOWN\n", + "'UNKNOWN' is invalid for 'version'", + id="invalid Version", + ), + pytest.param( + b"Metadata-Version: 2.2\nName: test-package\nVersion: UNKNOWN\n", + "'UNKNOWN' is invalid for 'version'", + id="invalid Version", + ), + pytest.param( + b"Metadata-Version: 2.3\n", + "'name' is a required field; 'version' is a required field", id="missing Name and Version", ), pytest.param( - b"Metadata-Version: 2.2\nName: UNKNOWN\nVersion: UNKNOWN\n", - "Name, Version", + b"Metadata-Version: 2.2\n", + "'name' is a required field; 'version' is a required field", id="missing Name and Version", ), pytest.param( - b"Metadata-Version: 2.3\nName: UNKNOWN\nVersion: 1.0.0\n", - "Name", + b"Metadata-Version: 2.3\nVersion: 1.0.0\n", + "'name' is a required field", id="missing Name", ), pytest.param( - b"Metadata-Version: 2.2\nName: UNKNOWN\nVersion: 1.0.0\n", - "Name", + b"Metadata-Version: 2.2\nVersion: 1.0.0\n", + "'name' is a required field", id="missing Name", ), pytest.param( - b"Metadata-Version: 2.3\nName: test-package\nVersion: UNKNOWN\n", - "Version", + b"Metadata-Version: 2.3\nName: test-package\n", + "'version' is a required field", id="missing Version", ), pytest.param( - b"Metadata-Version: 2.2\nName: test-package\nVersion: UNKNOWN\n", - "Version", + b"Metadata-Version: 2.2\nName: test-package\n", + "'version' is a required field", id="missing Version", ), + pytest.param( + b"Metadata-Version: 2.2\nName: test-package\nVersion: 1.0.0\nFoo: bar\n", + "unrecognized or malformed field 'foo'", + id="unrecognized field", + ), ], ) -def test_pkginfo_returns_no_metadata(read_data, missing_fields, monkeypatch): +def test_pkginfo_returns_no_metadata(read_data, exception_message, monkeypatch): """Raise an exception when pkginfo can't interpret the metadata.""" monkeypatch.setattr(package_file.wheel.Wheel, "read", lambda _: read_data) filename = "tests/fixtures/twine-1.5.0-py2.py3-none-any.whl" @@ -423,34 +447,7 @@ def test_pkginfo_returns_no_metadata(read_data, missing_fields, monkeypatch): with pytest.raises(exceptions.InvalidDistribution) as err: package_file.PackageFile.from_filename(filename, comment=None) - assert ( - f"Metadata is missing required fields: {missing_fields}." in err.value.args[0] - ) - - -def test_pkginfo_unrecognized_version(monkeypatch): - """Raise an exception when pkginfo doesn't recognize the version.""" - data = b"Metadata-Version: 102.3\nName: test-package\nVersion: 1.0.0\n" - monkeypatch.setattr(package_file.wheel.Wheel, "read", lambda _: data) - filename = "tests/fixtures/twine-1.5.0-py2.py3-none-any.whl" - - with pytest.raises(exceptions.InvalidDistribution) as err: - package_file.PackageFile.from_filename(filename, comment=None) - - assert "1.0, 1.1, 1.2, 2.0, 2.1, 2.2" in err.value.args[0] - - -def test_pkginfo_returns_no_metadata_py_below_1_11(monkeypatch): - """Raise special msg when pkginfo can't interpret metadata on pkginfo < 1.11.""" - data = b"Metadata-Version: 2.2\nName: UNKNOWN\nVersion: 1.0.0\n" - monkeypatch.setattr(package_file.wheel.Wheel, "read", lambda _: data) - monkeypatch.setattr(package_file.importlib_metadata, "version", lambda pkg: "1.10") - filename = "tests/fixtures/twine-1.5.0-py2.py3-none-any.whl" - - with pytest.raises(exceptions.InvalidDistribution) as err: - package_file.PackageFile.from_filename(filename, comment=None) - - assert "Make sure the distribution includes" in err.value.args[0] + assert exception_message in err.value.args[0] def test_malformed_from_file(monkeypatch): diff --git a/tests/test_repository.py b/tests/test_repository.py index e954c89d..dab2feac 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -18,6 +18,7 @@ import pytest import requests +from twine import package from twine import repository from twine import utils @@ -37,7 +38,7 @@ def test_gpg_signature_structure_is_preserved(): "gpg_signature": ("filename.asc", "filecontent"), } - tuples = repository.Repository._convert_data_to_list_of_tuples(data) + tuples = repository.Repository._convert_metadata_to_list_of_tuples(data) assert tuples == [("gpg_signature", ("filename.asc", "filecontent"))] @@ -47,17 +48,32 @@ def test_iterables_are_flattened(): "platform": ["UNKNOWN"], } - tuples = repository.Repository._convert_data_to_list_of_tuples(data) + tuples = repository.Repository._convert_metadata_to_list_of_tuples(data) assert tuples == [("platform", "UNKNOWN")] data = { "platform": ["UNKNOWN", "ANOTHERPLATFORM"], } - tuples = repository.Repository._convert_data_to_list_of_tuples(data) + tuples = repository.Repository._convert_metadata_to_list_of_tuples(data) assert tuples == [("platform", "UNKNOWN"), ("platform", "ANOTHERPLATFORM")] +def test_form_metadata_translation(monkeypatch): + """Verify that package metadata is correctly translated.""" + metadata = open("tests/fixtures/everything.metadata") + monkeypatch.setattr(package.wheel.Wheel, "read", lambda _: metadata.read()) + filename = "tests/fixtures/twine-1.5.0-py2.py3-none-any.whl" + data = package.PackageFile.from_filename( + filename, comment="comment" + ).metadata_dictionary() + + tuples = repository.Repository._convert_metadata_to_list_of_tuples(data) + for key, value in tuples: + assert isinstance(key, str) + assert isinstance(value, str) + + def test_set_client_certificate(default_repo): """Set client certificate for session.""" assert default_repo.session.cert is None @@ -93,7 +109,7 @@ def test_package_is_uploaded_404s(default_repo): default_repo.session = pretend.stub( get=lambda url, headers: response_with(status_code=404) ) - package = pretend.stub(safe_name="fake", metadata=pretend.stub(version="2.12.0")) + package = pretend.stub(safe_name="fake", version="2.12.0") assert default_repo.package_is_uploaded(package) is False @@ -105,7 +121,7 @@ def test_package_is_uploaded_200s_with_no_releases(default_repo): status_code=200, _content=b'{"releases": {}}', _content_consumed=True ), ) - package = pretend.stub(safe_name="fake", metadata=pretend.stub(version="2.12.0")) + package = pretend.stub(safe_name="fake", version="2.12.0") assert default_repo.package_is_uploaded(package) is False @@ -115,8 +131,8 @@ def test_package_is_uploaded_with_releases_using_cache(default_repo): default_repo._releases_json_data = {"fake": {"0.1": [{"filename": "fake.whl"}]}} package = pretend.stub( safe_name="fake", + version="0.1", basefilename="fake.whl", - metadata=pretend.stub(version="0.1"), ) assert default_repo.package_is_uploaded(package) is True @@ -133,8 +149,8 @@ def test_package_is_uploaded_with_releases_not_using_cache(default_repo): ) package = pretend.stub( safe_name="fake", + version="0.1", basefilename="fake.whl", - metadata=pretend.stub(version="0.1"), ) assert default_repo.package_is_uploaded(package, bypass_cache=True) is True @@ -151,8 +167,8 @@ def test_package_is_uploaded_different_filenames(default_repo): ) package = pretend.stub( safe_name="fake", + version="0.1", basefilename="foo.whl", - metadata=pretend.stub(version="0.1"), ) assert default_repo.package_is_uploaded(package) is False @@ -298,8 +314,7 @@ def test_upload_retry(tmpdir, default_repo, caplog): def test_release_urls(package_meta, repository_url, release_urls): """Generate a set of PyPI release URLs for a list of packages.""" packages = [ - pretend.stub(safe_name=name, metadata=pretend.stub(version=version)) - for name, version in package_meta + pretend.stub(safe_name=name, version=version) for name, version in package_meta ] repo = repository.Repository( diff --git a/tests/test_wheel.py b/tests/test_wheel.py index eae3bd95..0dedf6cc 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -75,7 +75,7 @@ def test_read_non_existent_wheel_file_name(): with pytest.raises( exceptions.InvalidDistribution, match=re.escape(f"No such file: {file_name}") ): - wheel.Wheel(file_name) + wheel.Wheel(file_name).read() def test_read_invalid_wheel_extension(): @@ -85,7 +85,7 @@ def test_read_invalid_wheel_extension(): exceptions.InvalidDistribution, match=re.escape(f"Not a known archive format for file: {file_name}"), ): - wheel.Wheel(file_name) + wheel.Wheel(file_name).read() def test_read_wheel_empty_metadata(tmpdir): @@ -100,4 +100,4 @@ def test_read_wheel_empty_metadata(tmpdir): f"No METADATA in archive or METADATA missing 'Metadata-Version': {whl_file}" ), ): - wheel.Wheel(whl_file) + wheel.Wheel(whl_file).read() diff --git a/twine/bdist.py b/twine/bdist.py new file mode 100644 index 00000000..947d278f --- /dev/null +++ b/twine/bdist.py @@ -0,0 +1,50 @@ +# Copyright 2024 Daniele Nicolodi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import zipfile +from typing import Optional + +from twine import distribution +from twine import exceptions + +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + +class BDist(distribution.Distribution): + + @property + def py_version(self) -> Optional[str]: + (dist,) = importlib_metadata.Distribution.discover(path=[self.filename]) + return dist.metadata["Version"] + + def read(self) -> bytes: + fqn = os.path.abspath(os.path.normpath(self.filename)) + if not os.path.exists(fqn): + raise exceptions.InvalidDistribution(f"No such file: {fqn}") + + with zipfile.ZipFile(fqn) as archive: + names = [name for name in archive.namelist() if name.endswith("/PKG-INFO")] + names.sort(key=lambda x: len(x.split("/"))) + for name in names: + data = archive.read(name) + if b"Metadata-Version" in data: + return data + + raise exceptions.InvalidDistribution( + f"No PKG-INFO in archive or PKG-INFO missing 'Metadata-Version': {fqn}" + ) diff --git a/twine/cli.py b/twine/cli.py index 749a6452..efee0432 100644 --- a/twine/cli.py +++ b/twine/cli.py @@ -77,7 +77,7 @@ def configure_output() -> None: def list_dependencies_and_versions() -> List[Tuple[str, str]]: deps = [ "keyring", - "pkginfo", + "packaging", "requests", "requests-toolbelt", "urllib3", diff --git a/twine/commands/check.py b/twine/commands/check.py index 6b0fe5c7..9e1f1bc1 100644 --- a/twine/commands/check.py +++ b/twine/commands/check.py @@ -18,7 +18,7 @@ import io import logging import re -from typing import Dict, List, Optional, Tuple, cast +from typing import Dict, List, Tuple import readme_renderer.rst from rich import print @@ -84,8 +84,8 @@ def _check_file( package = package_file.PackageFile.from_filename(filename, comment=None) metadata = package.metadata_dictionary() - description = cast(Optional[str], metadata["description"]) - description_content_type = cast(Optional[str], metadata["description_content_type"]) + description = metadata.get("description") + description_content_type = metadata.get("description_content_type") if description_content_type is None: warnings.append( diff --git a/twine/distribution.py b/twine/distribution.py new file mode 100644 index 00000000..5c8884c4 --- /dev/null +++ b/twine/distribution.py @@ -0,0 +1,26 @@ +# Copyright 2024 Daniele Nicolodi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + + +class Distribution: + def __init__(self, filename: str) -> None: + self.filename = filename + + def read(self) -> bytes: + raise NotImplementedError + + @property + def py_version(self) -> Optional[str]: + return None diff --git a/twine/package.py b/twine/package.py index 8bcec211..00a59f96 100644 --- a/twine/package.py +++ b/twine/package.py @@ -18,39 +18,22 @@ import os import re import subprocess -import sys -import warnings -from typing import ( - Any, - Dict, - Iterable, - List, - NamedTuple, - Optional, - Sequence, - Tuple, - Union, - cast, -) - -if sys.version_info >= (3, 10): - import importlib.metadata as importlib_metadata -else: - import importlib_metadata - -import packaging.version -import pkginfo +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +from packaging import metadata from rich import print +from twine import bdist from twine import exceptions +from twine import sdist from twine import wheel from twine import wininst DIST_TYPES = { "bdist_wheel": wheel.Wheel, "bdist_wininst": wininst.WinInst, - "bdist_egg": pkginfo.BDist, - "sdist": pkginfo.SDist, + "bdist_egg": bdist.BDist, + "sdist": sdist.SDist, } DIST_EXTENSIONS = { @@ -62,8 +45,6 @@ ".zip": "sdist", } -MetadataValue = Union[Optional[str], Sequence[str], Tuple[str, bytes]] - logger = logging.getLogger(__name__) @@ -78,11 +59,15 @@ def _safe_name(name: str) -> str: return re.sub("[^A-Za-z0-9.]+", "-", name) -class CheckedDistribution(pkginfo.Distribution): - """A Distribution whose name and version are confirmed to be defined.""" - - name: str - version: str +class PackageMetadata(metadata.RawMetadata, total=False): + pyversion: Optional[str] + filetype: Optional[str] + comment: Optional[str] + gpg_signature: Tuple[str, bytes] + attestations: str + md5_digest: str + sha256_digest: Optional[str] + blake2_256_digest: str class PackageFile: @@ -90,7 +75,7 @@ def __init__( self, filename: str, comment: Optional[str], - metadata: CheckedDistribution, + metadata: metadata.RawMetadata, python_version: Optional[str], filetype: Optional[str], ) -> None: @@ -100,7 +85,8 @@ def __init__( self.metadata = metadata self.python_version = python_version self.filetype = filetype - self.safe_name = _safe_name(metadata.name) + self.safe_name = _safe_name(metadata["name"]) + self.version: str = metadata["version"] self.signed_filename = self.filename + ".asc" self.signed_basefilename = self.basefilename + ".asc" self.gpg_signature: Optional[Tuple[str, bytes]] = None @@ -120,8 +106,9 @@ def from_filename(cls, filename: str, comment: Optional[str]) -> "PackageFile": for ext, dtype in DIST_EXTENSIONS.items(): if filename.endswith(ext): try: - with warnings.catch_warnings(record=True) as captured: - meta = DIST_TYPES[dtype](filename) + dist = DIST_TYPES[dtype](filename) + data = dist.read() + py_version = dist.py_version except EOFError: raise exceptions.InvalidDistribution( "Invalid distribution file: '%s'" % os.path.basename(filename) @@ -133,104 +120,42 @@ def from_filename(cls, filename: str, comment: Optional[str]) -> "PackageFile": "Unknown distribution format: '%s'" % os.path.basename(filename) ) - supported_metadata = list(pkginfo.distribution.HEADER_ATTRS) - if cls._is_unknown_metadata_version(captured): + # Parse and validate metadata. + meta, unparsed = metadata.parse_email(data) + if unparsed: raise exceptions.InvalidDistribution( - "Make sure the distribution is using a supported Metadata-Version: " - f"{', '.join(supported_metadata)}." + "Invalid distribution metadata: {}".format( + "; ".join( + f"unrecognized or malformed field {key!r}" for key in unparsed + ) + ) ) - # If pkginfo <1.11 encounters a metadata version it doesn't support, it may give - # back empty metadata. At the very least, we should have a name and version, - # which could also be empty if, for example, a MANIFEST.in doesn't include - # setup.cfg. - missing_fields = [ - f.capitalize() for f in ["name", "version"] if not getattr(meta, f) - ] - if missing_fields: - msg = f"Metadata is missing required fields: {', '.join(missing_fields)}." - if cls._pkginfo_before_1_11(): - msg += ( - "\n" - "Make sure the distribution includes the files where those fields " - "are specified, and is using a supported Metadata-Version: " - f"{', '.join(supported_metadata)}." + try: + metadata.Metadata.from_raw(meta) + except metadata.ExceptionGroup as group: + raise exceptions.InvalidDistribution( + "Invalid distribution metadata: {}".format( + "; ".join(sorted(str(e) for e in group.exceptions)) ) - raise exceptions.InvalidDistribution(msg) - - py_version: Optional[str] - if dtype == "bdist_egg": - (dist,) = importlib_metadata.Distribution.discover(path=[filename]) - py_version = dist.metadata["Version"] - elif dtype == "bdist_wheel": - py_version = cast(wheel.Wheel, meta).py_version - elif dtype == "bdist_wininst": - py_version = cast(wininst.WinInst, meta).py_version - else: - py_version = None - - return cls( - filename, comment, cast(CheckedDistribution, meta), py_version, dtype - ) - - @staticmethod - def _is_unknown_metadata_version( - captured: Iterable[warnings.WarningMessage], - ) -> bool: - NMV = getattr(pkginfo.distribution, "NewMetadataVersion", None) - return any(warning.category is NMV for warning in captured) + ) - @staticmethod - def _pkginfo_before_1_11() -> bool: - ver = packaging.version.Version(importlib_metadata.version("pkginfo")) - return ver < packaging.version.Version("1.11") + return cls(filename, comment, meta, py_version, dtype) - def metadata_dictionary(self) -> Dict[str, MetadataValue]: + def metadata_dictionary(self) -> PackageMetadata: """Merge multiple sources of metadata into a single dictionary. Includes values from filename, PKG-INFO, hashers, and signature. """ - meta = self.metadata - data: Dict[str, MetadataValue] = { - # identify release - "name": self.safe_name, - "version": meta.version, - # file content - "filetype": self.filetype, - "pyversion": self.python_version, - # additional meta-data - "metadata_version": meta.metadata_version, - "summary": meta.summary, - "home_page": meta.home_page, - "author": meta.author, - "author_email": meta.author_email, - "maintainer": meta.maintainer, - "maintainer_email": meta.maintainer_email, - "license": meta.license, - "description": meta.description, - "keywords": meta.keywords, - "platform": meta.platforms, - "classifiers": meta.classifiers, - "download_url": meta.download_url, - "supported_platform": meta.supported_platforms, - "comment": self.comment, - "sha256_digest": self.sha2_digest, - # PEP 314 - "provides": meta.provides, - "requires": meta.requires, - "obsoletes": meta.obsoletes, - # Metadata 1.2 - "project_urls": meta.project_urls, - "provides_dist": meta.provides_dist, - "obsoletes_dist": meta.obsoletes_dist, - "requires_dist": meta.requires_dist, - "requires_external": meta.requires_external, - "requires_python": meta.requires_python, - # Metadata 2.1 - "provides_extra": meta.provides_extras, - "description_content_type": meta.description_content_type, - # Metadata 2.2 - "dynamic": meta.dynamic, - } + data = PackageMetadata(**self.metadata) + + # override name with safe name + data["name"] = self.safe_name + # file content + data["pyversion"] = self.python_version + data["filetype"] = self.filetype + # additional meta-data + data["comment"] = self.comment + data["sha256_digest"] = self.sha2_digest if self.gpg_signature is not None: data["gpg_signature"] = self.gpg_signature diff --git a/twine/repository.py b/twine/repository.py index 9109756c..e95de450 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -25,6 +25,13 @@ import twine from twine import package as package_file +# Map ``packaging.metadata.RawMetadata`` fields to package upload form fields. +RAW_METADATA_TO_FORM_MAPPING = { + "license_files": "license_file", + "platforms": "platform", + "supported_platforms": "supported_platform", +} + LEGACY_PYPI = "https://pypi.python.org/" LEGACY_TEST_PYPI = "https://testpypi.python.org/" WAREHOUSE = "https://upload.pypi.org/" @@ -88,12 +95,23 @@ def close(self) -> None: self.session.close() @staticmethod - def _convert_data_to_list_of_tuples(data: Dict[str, Any]) -> List[Tuple[str, Any]]: - data_to_send = [] + def _convert_metadata_to_list_of_tuples( + data: package_file.PackageMetadata, + ) -> List[Tuple[str, Any]]: + # This does what ``warehouse.forklift.parse_form_metadata()`` does, in reverse. + data_to_send: List[Tuple[str, Any]] = [] for key, value in data.items(): + key = RAW_METADATA_TO_FORM_MAPPING.get(key, key) if key == "gpg_signature": assert isinstance(value, tuple) data_to_send.append((key, value)) + elif key == "project_urls": + assert isinstance(value, dict) + for name, url in value.items(): + data_to_send.append((key, f"{name}, {url}")) + elif key == "keywords": + assert isinstance(value, list) + data_to_send.append((key, ", ".join(value))) elif isinstance(value, (list, tuple)): data_to_send.extend((key, item) for item in value) else: @@ -110,12 +128,12 @@ def set_client_certificate(self, clientcert: Optional[str]) -> None: self.session.cert = clientcert def register(self, package: package_file.PackageFile) -> requests.Response: - data = package.metadata_dictionary() - data.update({":action": "submit", "protocol_version": "1"}) - print(f"Registering {package.basefilename}") - data_to_send = self._convert_data_to_list_of_tuples(data) + metadata = package.metadata_dictionary() + data_to_send = self._convert_metadata_to_list_of_tuples(metadata) + data_to_send.append((":action", "submit")) + data_to_send.append(("protocol_version", "1")) encoder = requests_toolbelt.MultipartEncoder(data_to_send) resp = self.session.post( self.url, @@ -128,19 +146,12 @@ def register(self, package: package_file.PackageFile) -> requests.Response: return resp def _upload(self, package: package_file.PackageFile) -> requests.Response: - data = package.metadata_dictionary() - data.update( - { - # action - ":action": "file_upload", - "protocol_version": "1", - } - ) - - data_to_send = self._convert_data_to_list_of_tuples(data) - print(f"Uploading {package.basefilename}") + metadata = package.metadata_dictionary() + data_to_send = self._convert_metadata_to_list_of_tuples(metadata) + data_to_send.append((":action", "file_upload")) + data_to_send.append(("protocol_version", "1")) with open(package.filename, "rb") as fp: data_to_send.append( ( @@ -227,7 +238,7 @@ def package_is_uploaded( releases = {} self._releases_json_data[safe_name] = releases - packages = releases.get(package.metadata.version, []) + packages = releases.get(package.version, []) for uploaded_package in packages: if uploaded_package["filename"] == package.basefilename: @@ -244,7 +255,7 @@ def release_urls(self, packages: List[package_file.PackageFile]) -> Set[str]: return set() return { - f"{url}project/{package.safe_name}/{package.metadata.version}/" + f"{url}project/{package.safe_name}/{package.version}/" for package in packages } diff --git a/twine/sdist.py b/twine/sdist.py new file mode 100644 index 00000000..be020451 --- /dev/null +++ b/twine/sdist.py @@ -0,0 +1,40 @@ +# Copyright 2024 Daniele Nicolodi +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import tarfile + +from twine import distribution +from twine import exceptions + + +class SDist(distribution.Distribution): + + def read(self) -> bytes: + fqn = os.path.abspath(os.path.normpath(self.filename)) + if not os.path.exists(fqn): + raise exceptions.InvalidDistribution(f"No such file: {fqn}") + + with tarfile.open(fqn, "r:gz") as sdist: + members = [m for m in sdist.getmembers() if m.name.endswith("/PKG-INFO")] + members.sort(key=lambda x: len(x.name.split("/"))) + for member in members: + fd = sdist.extractfile(member) + assert fd is not None, "for mypy" + data = fd.read() + if b"Metadata-Version" in data: + return data + + raise exceptions.InvalidDistribution( + f"No PKG-INFO in archive or PKG-INFO missing 'Metadata-Version': {fqn}" + ) diff --git a/twine/wheel.py b/twine/wheel.py index a2a8ba8a..430d0a3b 100644 --- a/twine/wheel.py +++ b/twine/wheel.py @@ -11,20 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import io import os import re import zipfile -from typing import List, Optional -from typing import cast as type_cast +from typing import List, Literal, Optional -from pkginfo import distribution +import packaging.metadata +from twine import distribution from twine import exceptions # Monkeypatch Metadata 2.0 support -distribution.HEADER_ATTRS_2_0 = distribution.HEADER_ATTRS_1_2 -distribution.HEADER_ATTRS.update({"2.0": distribution.HEADER_ATTRS_2_0}) +packaging.metadata._VALID_METADATA_VERSIONS = ["1.0", "1.1", "1.2", "2.0", "2.1", "2.2", "2.3", "2.4"] wheel_file_re = re.compile( @@ -36,15 +34,10 @@ class Wheel(distribution.Distribution): - def __init__(self, filename: str, metadata_version: Optional[str] = None) -> None: - self.filename = filename - self.basefilename = os.path.basename(self.filename) - self.metadata_version = metadata_version - self.extractMetadata() @property - def py_version(self) -> str: - wheel_info = wheel_file_re.match(self.basefilename) + def py_version(self) -> Optional[str]: + wheel_info = wheel_file_re.match(os.path.basename(self.filename)) if wheel_info is None: return "any" else: @@ -88,12 +81,3 @@ def read_file(name: str) -> bytes: "No METADATA in archive or METADATA missing 'Metadata-Version': " "%s (searched %s)" % (fqn, ",".join(searched_files)) ) - - def parse(self, data: bytes) -> None: - super().parse(data) - - fp = io.StringIO(data.decode("utf-8", errors="replace")) - # msg is ``email.message.Message`` which is a legacy API documented - # here: https://docs.python.org/3/library/email.compat32-message.html - msg = distribution.parse(fp) - self.description = type_cast(str, msg.get_payload()) diff --git a/twine/wininst.py b/twine/wininst.py index 42f0d9be..ca602077 100644 --- a/twine/wininst.py +++ b/twine/wininst.py @@ -3,21 +3,16 @@ import zipfile from typing import Optional -from pkginfo import distribution - +from twine import distribution from twine import exceptions wininst_file_re = re.compile(r".*py(?P\d+\.\d+)\.exe$") class WinInst(distribution.Distribution): - def __init__(self, filename: str, metadata_version: Optional[str] = None) -> None: - self.filename = filename - self.metadata_version = metadata_version - self.extractMetadata() @property - def py_version(self) -> str: + def py_version(self) -> Optional[str]: m = wininst_file_re.match(self.filename) if m is None: return "any"