Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Look for PyPI tokens in keyring when uploading #649

Merged
merged 4 commits into from
Nov 8, 2023
Merged

Look for PyPI tokens in keyring when uploading #649

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7da2813
Look for PyPI tokens in keyring when uploading
takluyver Jun 20, 2023
a87a1bc
Fix upload tests
takluyver Jun 21, 2023
364209e
Add tests for retrieving token from keyring
takluyver Jun 21, 2023
3e58d78
Rename is_warehouse -> is_pypi, make it a property
takluyver Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 104 additions & 64 deletions flit/upload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
import logging
import os
from pathlib import Path
import re
import requests
import sys
from dataclasses import dataclass
from typing import Optional
from urllib.parse import urlparse

from flit_core.common import Metadata
from flit_core.common import make_metadata, Metadata, Module
from .config import read_flit_config

log = logging.getLogger(__name__)

Expand All @@ -27,6 +31,15 @@
"http://upload.pypi.io/",
)


@dataclass
class RepoDetails:
url: str
username: Optional[str] = None
password: Optional[str] = None
is_warehouse: bool = True
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you want to update that naming to is_legacy?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is_pypi might be a better name now - it's distinguishing PyPI (the new site, aka Warehouse) from other servers like devpi, and originally from PyPI classic.

It's only used for the 'Package is at' log message, so it's not too important that it's always right.



def get_repositories(file="~/.pypirc"):
"""Get the known repositories from a pypirc file.

Expand All @@ -38,9 +51,7 @@ def get_repositories(file="~/.pypirc"):
file = os.path.expanduser(file)

if not os.path.isfile(file):
return {'pypi': {
'url': PYPI, 'username': None, 'password': None,
}}
return {'pypi': RepoDetails(url=PYPI)}

cp.read(file)
else:
Expand All @@ -51,16 +62,16 @@ def get_repositories(file="~/.pypirc"):
repos = {}

for name in names:
repos[name] = {
'url': cp.get(name, 'repository', fallback=PYPI),
'username': cp.get(name, 'username', fallback=None),
'password': cp.get(name, 'password', fallback=None),
}
repos[name] = RepoDetails(
url=cp.get(name, 'repository', fallback=PYPI),
username=cp.get(name, 'username', fallback=None),
password=cp.get(name, 'password', fallback=None),
)

return repos


def get_repository(pypirc_path="~/.pypirc", name=None):
def get_repository(pypirc_path="~/.pypirc", name=None, project_name=None):
"""Get the url, username and password for one repository.

Returns a dict with keys 'url', 'username', 'password'.
Expand All @@ -74,55 +85,64 @@ def get_repository(pypirc_path="~/.pypirc", name=None):
4. Default PyPI (hardcoded)

Username:
1. Command line arg --repository (looked up in .pypirc)
2. $FLIT_USERNAME
3. Repository called 'pypi' from .pypirc
4. Terminal prompt (write to .pypirc if it doesn't exist yet)
1. $FLIT_USERNAME
2. Repository with specified name (default 'pypi') in .pypirc
3. Terminal prompt (write to .pypirc if it doesn't exist yet)

Password:
1. Command line arg --repository (looked up in .pypirc)
2. $FLIT_PASSWORD
3. Repository called 'pypi' from .pypirc
3. keyring
4. Terminal prompt (store to keyring if available)
1. $FLIT_PASSWORD
2. Repository with specified name (default 'pypi') in .pypirc
3. keyring - pypi_token:project:<project_name>
4. keyring - pypi_token:user:<username>
5. keyring - username
6. Terminal prompt (store to keyring if available)
"""
log.debug("Loading repositories config from %r", pypirc_path)
repos_cfg = get_repositories(pypirc_path)

if name is not None:
repo = repos_cfg[name]
if 'FLIT_INDEX_URL' in os.environ:
raise EnvironmentError(
"Use either FLIT_INDEX_URL or --repository, not both"
)
elif 'FLIT_INDEX_URL' in os.environ:
repo = {'url': os.environ['FLIT_INDEX_URL'],
'username': None, 'password': None}
repo = RepoDetails(url=os.environ['FLIT_INDEX_URL'])
elif 'pypi' in repos_cfg:
repo = repos_cfg['pypi']

if 'FLIT_PASSWORD' in os.environ:
repo['password'] = os.environ['FLIT_PASSWORD']
else:
repo = {'url': PYPI, 'username': None, 'password': None}
repo = RepoDetails(url=PYPI)

if repo['url'].startswith(SWITCH_TO_HTTPS):
if repo.url.startswith(SWITCH_TO_HTTPS):
# Use https for PyPI, even if an http URL was given
repo['url'] = 'https' + repo['url'][4:]
elif repo['url'].startswith('http://'):
repo.url = 'https' + repo.url[4:]
elif repo.url.startswith('http://'):
log.warning("Unencrypted connection - credentials may be visible on "
"the network.")
log.info("Using repository at %s", repo['url'])

if ('FLIT_USERNAME' in os.environ) and ((name is None) or (not repo['username'])):
repo['username'] = os.environ['FLIT_USERNAME']
if sys.stdin.isatty():
while not repo['username']:
repo['username'] = input("Username: ")
if repo['url'] == PYPI:
repo.is_warehouse = repo.url.rstrip('/').endswith('/legacy')
log.info("Using repository at %s", repo.url)

if 'FLIT_USERNAME' in os.environ:
repo.username = os.environ['FLIT_USERNAME']
if not repo.username and sys.stdin.isatty():
while not repo.username:
repo.username = input("Username: ")
if repo.url == PYPI:
write_pypirc(repo, pypirc_path)
elif not repo['username']:
elif not repo.username:
raise Exception("Could not find username for upload.")

repo['password'] = get_password(repo, prefer_env=(name is None))
if 'FLIT_PASSWORD' in os.environ:
repo.password = os.environ['FLIT_PASSWORD']

repo['is_warehouse'] = repo['url'].rstrip('/').endswith('/legacy')
if not repo.password:
token = find_token(repo, project_name)
if token is not None:
repo.username = '__token__'
repo.password = token
else:
repo.password = get_password(repo)

return repo

Expand All @@ -135,23 +155,17 @@ def write_pypirc(repo, file="~/.pypirc"):

with open(file, 'w', encoding='utf-8') as f:
f.write("[pypi]\n"
"username = %s\n" % repo['username'])

def get_password(repo, prefer_env):
if ('FLIT_PASSWORD' in os.environ) and (prefer_env or not repo['password']):
return os.environ['FLIT_PASSWORD']

if repo['password']:
return repo['password']
"username = %s\n" % repo.username)

def get_password(repo: RepoDetails):
try:
import keyring, keyring.errors
except ImportError: # pragma: no cover
log.warning("Install keyring to store passwords securely")
log.warning("Install keyring to store tokens/passwords securely")
keyring = None
else:
try:
stored_pw = keyring.get_password(repo['url'], repo['username'])
stored_pw = keyring.get_password(repo.url, repo.username)
if stored_pw is not None:
return stored_pw
except keyring.errors.KeyringError as e:
Expand All @@ -160,21 +174,44 @@ def get_password(repo, prefer_env):
if sys.stdin.isatty():
pw = None
while not pw:
print('Server :', repo['url'])
print('Username:', repo['username'])
print('Server :', repo.url)
print('Username:', repo.username)
pw = getpass.getpass('Password: ')
else:
raise Exception("Could not find password for upload.")

if keyring is not None:
try:
keyring.set_password(repo['url'], repo['username'], pw)
keyring.set_password(repo.url, repo.username, pw)
log.info("Stored password with keyring")
except keyring.errors.KeyringError as e:
log.warning("Could not store password in keyring (%s)", e)

return pw


def find_token(repo: RepoDetails, project_name: str):
# https://packaging.python.org/en/latest/specifications/name-normalization/
project_name = re.sub(r"[-_.]+", "-", project_name).lower()
candidate_keys = [f"pypi_token:project:{project_name}"]

if repo.username is not None:
candidate_keys.append(f"pypi_token:user:{repo.username}")

try:
import keyring, keyring.errors
except ImportError: # pragma: no cover
pass
else:
try:
for key in candidate_keys:
token = keyring.get_password(repo.url, key)
if token is not None:
return token
except keyring.errors.KeyringError as e:
log.warning("Could not get token from keyring (%s)", e)


def build_post_data(action, metadata:Metadata):
"""Prepare the metadata needed for requests to PyPI.
"""
Expand Down Expand Up @@ -217,7 +254,7 @@ def build_post_data(action, metadata:Metadata):

return {k:v for k,v in d.items() if v}

def upload_file(file:Path, metadata:Metadata, repo):
def upload_file(file:Path, metadata:Metadata, repo: RepoDetails):
"""Upload a file to an index server, given the index server details.
"""
data = build_post_data('file_upload', metadata)
Expand All @@ -237,27 +274,24 @@ def upload_file(file:Path, metadata:Metadata, repo):
data['sha256_digest'] = hashlib.sha256(content).hexdigest()

log.info('Uploading %s...', file)
resp = requests.post(repo['url'],
data=data,
files=files,
auth=(repo['username'], repo['password']),
)
resp = requests.post(
repo.url, data=data, files=files, auth=(repo.username, repo.password),
)
resp.raise_for_status()


def do_upload(file:Path, metadata:Metadata, pypirc_path="~/.pypirc", repo_name=None):
def do_upload(file:Path, metadata:Metadata, repo: RepoDetails):
"""Upload a file to an index server.
"""
repo = get_repository(pypirc_path, repo_name)
upload_file(file, metadata, repo)

if repo['is_warehouse']:
domain = urlparse(repo['url']).netloc
if repo.is_warehouse:
domain = urlparse(repo.url).netloc
if domain.startswith('upload.'):
domain = domain[7:]
log.info("Package is at https://%s/project/%s/", domain, metadata.name)
else:
log.info("Package is at %s/%s", repo['url'], metadata.name)
log.info("Package is at %s/%s", repo.url, metadata.name)


def main(ini_path, repo_name, pypirc_path=None, formats=None, gen_setup_py=True,
Expand All @@ -268,12 +302,18 @@ def main(ini_path, repo_name, pypirc_path=None, formats=None, gen_setup_py=True,
elif not os.path.isfile(pypirc_path):
raise FileNotFoundError("The specified pypirc config file does not exist.")

ini_info = read_flit_config(ini_path)
srcdir = ini_path.parent
module = Module(ini_info.module, srcdir)
metadata = make_metadata(module, ini_info)
repo = get_repository(pypirc_path, repo_name, project_name=metadata.name)

from . import build
built = build.main(
ini_path, formats=formats, gen_setup_py=gen_setup_py, use_vcs=use_vcs
)

if built.wheel is not None:
do_upload(built.wheel.file, built.wheel.builder.metadata, pypirc_path, repo_name)
do_upload(built.wheel.file, built.wheel.builder.metadata, repo)
if built.sdist is not None:
do_upload(built.sdist.file, built.sdist.builder.metadata, pypirc_path, repo_name)
do_upload(built.sdist.file, built.sdist.builder.metadata, repo)
Loading