ENH: backport scipy changes to openblas download script

Author: mattip

azure-pipelines.yml

@@ -40,7 +40,6 @@ stages:
             /bin/bash -xc "cd numpy && \
             /opt/python/cp38-cp38/bin/python -mvenv venv &&\
             source venv/bin/activate && \
-            python3 -m pip install urllib3 && \
             target=\$(python3 tools/openblas_support.py) && \
             cp -r \$target/lib/* /usr/lib && \
             cp \$target/include/* /usr/include && \
@@ -122,7 +121,6 @@ stages:
     # primarily on file size / name details
     - script: |
         set -xe
-        python -m pip install urllib3
         target=$(python tools/openblas_support.py)
         ls -lR $target
         # manually link to appropriate system paths

azure-steps-windows.yml

@@ -14,7 +14,7 @@ steps:
     # to $PYTHON_EXE's directory since that is on a different drive which
     # mingw does not like. Instead copy it to a directory and set OPENBLAS,
     # since OPENBLAS will be picked up by the openblas discovery
-    python -m pip  install urllib3
+    python -m pip  install
     $target = $(python tools/openblas_support.py)
     mkdir openblas
     echo Copying $target to openblas/openblas$env:OPENBLAS_SUFFIX.a

shippable.yml

@@ -25,7 +25,6 @@ build:
     - sudo add-apt-repository ppa:ubuntu-toolchain-r/test
     - sudo apt-get update
     - sudo apt-get install gcc gfortran libgfortran5
-    - python -mpip install urllib3
     - target=$(python tools/openblas_support.py)
     - ls -lR "${target}"
     - sudo cp -r "${target}"/lib/* /usr/lib

tools/openblas_support.py

@@ -1,22 +1,41 @@
+import glob
+import hashlib
 import os
+import platform
 import sys
-import glob
 import shutil
+import tarfile
 import textwrap
-import platform
+import zipfile
 
 from tempfile import mkstemp, gettempdir
-import zipfile
-import tarfile
+from urllib.request import urlopen, Request
 
 OPENBLAS_V = '0.3.9'
 # Temporary build of OpenBLAS to test a fix for dynamic detection of CPU
 OPENBLAS_LONG = 'v0.3.7-527-g79fd006c'  # the 0.3.7 is misleading
 BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs'
 BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download'
 ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', 'i686', 'ppc64le', 's390x']
+sha256_vals = {
+"openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840",
+"openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17",
+"openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259",
+"openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22",
+"openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8",
+"openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0",
+"openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439",
+"openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea",
+"openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240",
+"openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814",
+}
+
 
 IS_32BIT = sys.maxsize < 2**32
+
 def get_arch():
     if platform.system() == 'Windows':
         ret = 'windows'
@@ -50,10 +69,12 @@ def get_manylinux(arch):
 
 
 def download_openblas(target, arch, ilp64):
-    import urllib3
     ml_ver = get_manylinux(arch)
     fnsuffix = {None: "", "64_": "64_"}[ilp64]
     filename = ''
+    headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; '
+                              '(KHTML, like Gecko) Chrome/41.0.2228.0 '
+                              'Safari/537.3')}
     if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'):
         suffix = f'manylinux{ml_ver}_{arch}.tar.gz'
         filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
@@ -71,15 +92,26 @@ def download_openblas(target, arch, ilp64):
         typ = 'zip'
     if not filename:
         return None
-    print("Downloading:", filename, file=sys.stderr)
-    http = urllib3.PoolManager()
-    response = http.request('GET', filename)
+    req = Request(url=filename, headers=headers)
+    response = urlopen(req)
+    length = response.getheader('content-length')
     if response.status != 200:
         print(f'Could not download "{filename}"', file=sys.stderr)
         return None
+    print(f"Downloading {length} from {filename}", file=sys.stderr)
+    data = response.read()
+    # Verify hash
+    key = os.path.basename(filename)
+    sha256_returned = hashlib.sha256(data).hexdigest()
+    if key not in sha256_vals:
+        raise ValueError(
+            f'key "{key}" with hash "{sha256_returned}" not in sha256_vals')
+    sha256_expected = sha256_vals[key]
+    if sha256_returned != sha256_expected:
+        raise ValueError(f'sha256 hash mismatch for filename {filename}')
     print("Saving to file", file=sys.stderr)
     with open(target, 'wb') as fid:
-        fid.write(response.data)
+        fid.write(data)
     return typ
 
 def setup_openblas(arch=get_arch(), ilp64=get_ilp64()):
@@ -201,9 +233,10 @@ def test_setup(arches):
     def items():
         for arch in arches:
             yield arch, None
-            if arch in ('x86', 'darwin', 'windows'):
+            if arch not in ('i686'):
                 yield arch, '64_'
 
+    errs = []
     for arch, ilp64 in items():
         if arch == '':
             continue
@@ -212,9 +245,11 @@ def items():
         try:
             try:
                 target = setup_openblas(arch, ilp64)
-            except:
-                print(f'Could not setup {arch}')
-                raise
+            except Exception as e:
+                print(f'Could not setup {arch}:')
+                print(str(e))
+                errs.append(e)
+                continue
             if not target:
                 raise RuntimeError(f'Could not setup {arch}')
             print(target)
@@ -231,6 +266,9 @@ def items():
                     os.unlink(target)
                 else:
                     shutil.rmtree(target)
+    if errs:
+        raise errs[0]
+
 
 def test_version(expected_version, ilp64=get_ilp64()):
     """

tools/pypy-test.sh

@@ -6,9 +6,10 @@ set -o pipefail
 # Print expanded commands
 set -x
 
-#sudo apt-get -yq update
-#sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 python3-urllib3
-F77=gfortran-5 F90=gfortran-5 \
+sudo apt-get -yq update
+sudo apt-get -yq install gfortran-5
+export F77=gfortran-5
+export F90=gfortran-5
 
 # Download the proper OpenBLAS x64 precompiled library
 target=$(python3 tools/openblas_support.py)

tools/travis-before-install.sh

@@ -41,8 +41,7 @@ pip install --upgrade pip
 # A specific version of cython is required, so we read the cython package
 # requirement using `grep cython test_requirements.txt` instead of simply
 # writing 'pip install setuptools wheel cython'.
-# urllib3 is needed for openblas_support
-pip install setuptools wheel urllib3 `grep cython test_requirements.txt`
+pip install setuptools wheel `grep cython test_requirements.txt`
 
 if [ -n "$DOWNLOAD_OPENBLAS" ]; then
   pwd