Merge tag '1.1.3'

Bump version: 1.1.2 → 1.1.3

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.1
+current_version = 1.1.3
 tag_name = {new_version}
 files = setup.py doc/conf.py pypicloud/__init__.py
 commit = True

CHANGES.rst

@@ -2,6 +2,15 @@ Changelog
 =========
 If you are upgrading an existing installation, read :ref:`the instructions <upgrade>`
 
+1.1.3 - 2020/8/17
+-----------------
+* Fix metadata storage issue with some S3-compatible backends (:pr:`255`)
+* Command line arg to generate password hash from stdin (:pr:`253`)
+
+1.1.2 - 2020/7/23
+-----------------
+* Fix error when package in local storage but not in fallback repository (:issue:`251`)
+
 1.1.1 - 2020/6/14
 -----------------
 * Fix an exception when ``pypi.use_json_scraper = false`` (:issue:`250`)

doc/changes.rst

@@ -2,6 +2,15 @@ Changelog
 =========
 If you are upgrading an existing installation, read :ref:`the instructions <upgrade>`
 
+1.1.3 - 2020/8/17
+-----------------
+* Fix metadata storage issue with some S3-compatible backends (:pr:`255`)
+* Command line arg to generate password hash from stdin (:pr:`253`)
+
+1.1.2 - 2020/7/23
+-----------------
+* Fix error when package in local storage but not in fallback repository (:issue:`251`)
+
 1.1.1 - 2020/6/14
 -----------------
 * Fix an exception when ``pypi.use_json_scraper = false`` (:issue:`250`)

doc/conf.py

@@ -24,7 +24,7 @@
 copyright = u'2013, Steven Arcangeli'
 github_user = u'stevearc'
 
-release = '1.1.1'
+release = '1.1.3'
 version = '.'.join(release.split('.')[:2])
 
 exclude_patterns = ['_build']

pypicloud/__init__.py

@@ -13,7 +13,7 @@
 from .locator import FormattedScrapingLocator, SimpleJsonLocator
 from .route import Root
 
-__version__ = "1.1.1"
+__version__ = "1.1.3"
 LOG = logging.getLogger(__name__)
 
 

pypicloud/locator.py

@@ -12,7 +12,7 @@ class SimpleJsonLocator(object):
 
     def __init__(self, base_index):
         self.base_index = base_index
-        # 10m cache
+        # 16m cache
         self._cache = TimedCache(1000, self._get_releases)
 
     def get_releases(self, project_name):
@@ -21,6 +21,9 @@ def get_releases(self, project_name):
     def _get_releases(self, project_name):
         url = "%s/pypi/%s/json" % (self.base_index, project_name)
         response = requests.get(url)
+        # Return empty list for 4xx
+        if 400 <= response.status_code < 500:
+            return []
         response.raise_for_status()
         data = response.json()
         items = []

pypicloud/models.py

@@ -72,6 +72,10 @@ def read_metadata(blob):
         metadata = {}
         for field in METADATA_FIELDS:
             value = blob.get(field)
+            if value:
+                metadata[field] = value
+                continue
+            value = blob.get(field.replace("_", "-"))
             if value:
                 metadata[field] = value
         return metadata

pypicloud/scripts.py

@@ -21,6 +21,7 @@ def gen_password(argv=None):
     if argv is None:
         argv = sys.argv[1:]
     parser = argparse.ArgumentParser(gen_password.__doc__)
+    parser.add_argument("-i", help="Read password from stdin", action="store_true")
     parser.add_argument("-r", help="Number of rounds", type=int)
     parser.add_argument(
         "-s",
@@ -29,21 +30,29 @@ def gen_password(argv=None):
         choices=SCHEMES,
     )
     args = parser.parse_args(argv)
-    print(_gen_password(args.s, args.r))
+    if args.i:
+        password = sys.stdin.readline()
+    else:
+        password = _get_password()
+    print(_gen_password(password, args.s, args.r))
 
 
-def _gen_password(scheme=None, rounds=None):
+def _get_password():
     """ Prompt user for a password twice for safety """
-    pwd_context = get_pwd_context(scheme, rounds)
     while True:
         password = getpass.getpass()
         verify = getpass.getpass()
         if password == verify:
-            return pwd_context.hash(password)
+            return password
         else:
             print("Passwords do not match!")
 
 
+def _gen_password(password: str, scheme: str = None, rounds: int = None) -> str:
+    pwd_context = get_pwd_context(scheme, rounds)
+    return pwd_context.hash(password)
+
+
 NO_DEFAULT = object()
 
 
@@ -198,7 +207,7 @@ def make_config(argv=None):
     data["validate_key"] = b64encode(os.urandom(32)).decode("utf-8")
 
     data["admin"] = prompt("Admin username?")
-    data["password"] = _gen_password()
+    data["password"] = _gen_password(_get_password())
 
     data["session_secure"] = env == "prod"
     data["env"] = env

pypicloud/storage/azure_blob.py

@@ -132,7 +132,7 @@ def upload(self, package, datastream):
         metadata = package.get_metadata()
         metadata["name"] = package.name
         metadata["version"] = package.version
-        normalize_metadata(metadata)
+        metadata = normalize_metadata(metadata)
 
         blob_client = self.container_client.get_blob_client(blob=path)
         blob_client.upload_blob(data=datastream, metadata=metadata)

pypicloud/storage/s3.py

@@ -194,7 +194,7 @@ def upload(self, package, datastream):
         metadata = package.get_metadata()
         metadata["name"] = package.name
         metadata["version"] = package.version
-        normalize_metadata(metadata)
+        metadata = normalize_metadata(metadata)
         key.put(Metadata=metadata, Body=datastream, **kwargs)
 
     def delete(self, package):

pypicloud/util.py

@@ -44,16 +44,24 @@ def normalize_name(name: str) -> str:
     return re.sub(r"[-_.]+", "-", name).lower()
 
 
-def normalize_metadata(metadata: Dict[str, Union[str, bytes]]) -> None:
-    """Strip non-ASCII characters from metadata"""
-    for key, value in metadata.items():
-        if isinstance(value, bytes):
-            value = value.decode("utf-8")
-
-        if isinstance(value, str):
-            metadata[key] = "".join(
-                c for c in unicodedata.normalize("NFKD", value) if ord(c) < 128
-            )
+def normalize_metadata_value(value: Union[str, bytes]) -> str:
+    """Strip non-ASCII characters from metadata values"""
+    if isinstance(value, bytes):
+        value = value.decode("utf-8")
+    if isinstance(value, str):
+        value = "".join(c for c in unicodedata.normalize("NFKD", value) if ord(c) < 128)
+    return value
+
+
+def normalize_metadata(metadata: Dict[str, Union[str, bytes]]) -> Dict[str, str]:
+    """
+    Strip non-ASCII characters from metadata values
+    and replace "_" in metadata keys to "-"
+    """
+    return {
+        key.replace("_", "-"): normalize_metadata_value(value)
+        for key, value in metadata.items()
+    }
 
 
 def create_matcher(queries: List[str], query_type: str) -> Callable[[str], bool]:

setup.py

@@ -52,14 +52,20 @@
 ]
 
 EXTRAS["server"] = ["waitress"]
-EXTRAS["lint"] = ["black", "pylint==2.3.1", "mypy", "sqlalchemy-stubs", "isort"]
+EXTRAS["lint"] = [
+    "black",
+    "pylint==2.3.1",
+    "mypy",
+    "sqlalchemy-stubs",
+    "isort>=4.2.5,<5",
+]
 EXTRAS["doc"] = ["numpydoc", "sphinx", "sphinx_rtd_theme"]
 
 
 if __name__ == "__main__":
     setup(
         name="pypicloud",
-        version="1.1.1",
+        version="1.1.3",
         description="Private PyPI backed by S3",
         long_description=README + "\n\n" + CHANGES,
         classifiers=[

tests/test_scripts.py

@@ -4,7 +4,6 @@
 from mock import patch
 
 from pypicloud import scripts
-from pypicloud.access import get_pwd_context
 
 
 class TestScripts(unittest.TestCase):
@@ -14,16 +13,16 @@ class TestScripts(unittest.TestCase):
     @patch.object(scripts, "getpass")
     def test_gen_password(self, getpass):
         """ Generate a password """
-        pwd_context = get_pwd_context()
         passwds = ["foo", "foo", "bar", "baz"]
         getpass.getpass.side_effect = passwds.pop
-        ret = scripts._gen_password()
+        ret = scripts._get_password()
         self.assertEqual(len(passwds), 0)
-        self.assertTrue(pwd_context.verify("foo", ret))
+        self.assertEqual(ret, "foo")
 
-    @patch.object(scripts, "_gen_password")
-    def test_cli_gen_password(self, genpass):
+    @patch.object(scripts, "_get_password")
+    def test_cli_get_password(self, genpass):
         """ Commandline prints generated password """
+        genpass.return_value = "foo"
         scripts.gen_password([])
         self.assertTrue(genpass.called)
 

tests/test_storage.py

@@ -117,7 +117,9 @@ def test_upload(self):
         self.assertEqual(key.metadata["name"], package.name)
         self.assertEqual(key.metadata["version"], package.version)
         self.assertEqual(key.metadata["summary"], package.summary)
-        self.assertDictContainsSubset(package.get_metadata(), key.metadata)
+        self.assertDictContainsSubset(
+            package.get_metadata(), Package.read_metadata(key.metadata)
+        )
 
     def test_upload_prepend_hash(self):
         """ If prepend_hash = True, attach a hash to the file path """

tox.ini

@@ -3,7 +3,8 @@ envlist = py35, py36, py37, py38, lint
 
 [testenv]
 deps =
-  python-dateutil<2.8.1 # Mandatory until botocore removes this requirement
+# Mandatory until botocore removes this requirement
+  python-dateutil<2.8.1
 setenv =
   BOTO_CONFIG = /nowhere
 extras =