bootstrap.py

#!/usr/bin/env python
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import argparse
import contextlib
import glob
import logging
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import time

from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL
from util import build_manifest, filter_deps, read_python_literal, \
                 merge_deps, print_deps, platform_tag

LOGGER = logging.getLogger(__name__)

# /path/to/infra
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*'


class NoWheelException(Exception):
  def __init__(self, name, version, build, source_sha):
    super(NoWheelException, self).__init__(
        'No matching wheel found for (%s==%s (build %s_%s))' %
        (name, version, build, source_sha))


def check_pydistutils():
  if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')):
    print(
        '\n'.join([
            '', '', '=========== ERROR ===========',
            'You have a ~/.pydistutils.cfg file, which interferes with the ',
            'infra virtualenv environment. Please move it to the side and ',
            'bootstrap again. Once infra has bootstrapped, you may move it '
            'back.', '',
            'Upstream bug: https://github.com/pypa/virtualenv/issues/88/', ''
        ]),
        file=sys.stderr)
    sys.exit(1)


def ls(prefix):
  from pip._vendor import requests  # pylint: disable=E0611
  for retry in range(4):
    try:
      r = requests.get(STORAGE_URL, params=dict(
          prefix=prefix,
          fields='items(name,md5Hash)'
      ))
      r.raise_for_status()
      data = r.json()
      break
    except (requests.exceptions.SSLError, requests.exceptions.HTTPError) as ex:
      delay = 4 ** (retry-1)
      print(
          'caught an error: %s: retrying in %f sec' % (ex, delay),
          file=sys.stderr)
      time.sleep(delay)
      continue
  else:
    raise Exception("exceeded allowed retries!")
  entries = data.get('items', [])
  for entry in entries:
    entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex')
    entry['local'] = False
    entry['link'] = OBJECT_URL.format(entry['name'], entry['md5Hash'])
  # Also look in the local cache
  entries.extend([
    {
      'name': fname,
      'md5Hash': None,
      'local': True,
      'link': LOCAL_OBJECT_URL.format(fname),
    }
    for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH,
                                        prefix.split('/')[-1] + '*'))])
  return entries


def sha_for(deps_entry):
  if 'rev' in deps_entry:
    return deps_entry['rev']
  else:
    return deps_entry['gs'].split('.')[0]


def get_links(deps):
  import pip._internal.utils.compatibility_tags as compatibility_tags
  import pip._internal.models.wheel as pip_wheel

  plat_tag = platform_tag()  # this is something like '_Ubuntu_14.04' or ''
  pep425_tags = compatibility_tags.get_supported()
  links = []

  for name, dep in deps.iteritems():
    version, source_sha = dep['version'] , sha_for(dep)
    prefix = '{}-{}-{}_{}'.format(name, version, dep['build'], source_sha)

    generic_wheels = []
    platform_wheels = []
    local_wheels = []

    for entry in ls('wheels/'+prefix):
      fname = entry['name'].split('/')[-1]
      wheel_info = pip_wheel.Wheel.wheel_file_re.match(fname)
      if not wheel_info:
        LOGGER.warning('Skipping invalid wheel: %r', fname)
        continue

      # This check skips all obviously unsupported wheels (like Linux wheels on
      # Windows).
      if not pip_wheel.Wheel(fname).supported(pep425_tags):
        continue

      if entry['local']:
        # A locally built wheel?
        local_wheels.append(entry)
      elif plat_tag and fname.startswith(prefix + plat_tag):
        # A wheel targeting our very specific platform (if any)? This is hit on
        # different versions of Ubuntu for example.
        platform_wheels.append(entry)
      else:
        # Some more generic wheel (e.g. 'linux1many' or source wheel).
        generic_wheels.append(entry)

    # Prefer local wheels if have them, then per-platform, then generic.
    wheel_set = local_wheels or platform_wheels or generic_wheels
    if not wheel_set:
      raise NoWheelException(name, version, dep['build'], source_sha)

    if len(wheel_set) != 1:
      LOGGER.warning('Letting pip choose a wheel for "%s":', name)
      for entry in wheel_set:
        LOGGER.warning(' * %s', entry['name'])

    links.extend(entry['link'] for entry in wheel_set)

  return links


@contextlib.contextmanager
def html_index(links):
  tf = tempfile.mktemp('.html')
  try:
    with open(tf, 'w') as f:
      print('<html><body>', file=f)
      for link in links:
        print('<a href="%s">wat</a>' % link, file=f)
      print('</body></html>', file=f)
    yield tf
  finally:
    os.unlink(tf)


def install(deps):
  if sys.platform.startswith('win'):
    # On Windows, "pip" is installed as a standalone binary called "pip.exe".
    pip = [os.path.join(sys.prefix, 'Scripts', 'pip')]
  else:
    # On Linux, "pip" is a "#!/...python"-bootstrapped wrapper. Because of
    # shebang length limitations, we will manually run this through the
    # Python interpreter rather than relying on shebang interpretation.
    pip = [
        os.path.join(sys.prefix, 'bin', 'python'),
        os.path.join(sys.prefix, 'bin', 'pip'),
    ]

  links = get_links(deps)
  with html_index(links) as ipath:
    requirements = []
    # TODO(iannucci): Do this as a requirements.txt
    for name, deps_entry in deps.iteritems():
      if not deps_entry.get('implicit'):
        requirements.append('%s==%s' % (name, deps_entry['version']))
    subprocess.check_call(
        pip + ['install', '--no-index', '-f', ipath] + requirements)


def activate_env(env, manifest, quiet=False):
  if not quiet:
    print('Activating environment: %r' % env)
  assert isinstance(manifest, dict)

  manifest_path = os.path.join(env, 'manifest.pyl')
  cur_manifest = read_python_literal(manifest_path)
  if cur_manifest != manifest:
    if not quiet:
      print('  Removing old environment: %r' % cur_manifest)
    shutil.rmtree(env, ignore_errors=True)
    cur_manifest = None

  virtualenv_dir = os.path.join(os.path.dirname(__file__), 'virtualenv-ext')
  os.environ.pop('PYTHONPATH', None)
  virtualenv_py = os.path.join(virtualenv_dir, 'virtualenv.py')

  # This script itself is run in a virtualenv, and weird behavior can
  # result if our version of virtualenv is of a different version than
  # virtualenv-ext/. To avoid this, execute virtualenv.py under the real
  # Python interpreter.
  real_python = sys.executable
  if hasattr(sys, 'real_prefix'):
    real_python = sys.real_prefix
    if not real_python.endswith('bin'):
      real_python = os.path.join(real_python, 'bin')
    real_python = os.path.join(real_python, 'python')
    if sys.platform.startswith('win'):
      real_python += '.exe'

  if cur_manifest is None:
    check_pydistutils()

    if not quiet:
      print('  Building new environment')
    subprocess.check_call([real_python, virtualenv_py, env, '--no-download'])

  if not quiet:
    print('  Activating environment')
  bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin'
  activate_this = os.path.join(env, bin_dir, 'activate_this.py')
  execfile(activate_this, dict(__file__=activate_this))

  if cur_manifest is None:
    deps = manifest.get('deps', {})
    if not quiet:
      print('  Installing deps')
      print_deps(deps, indent=2, with_implicit=False)
    install(deps)
    subprocess.check_call([real_python, virtualenv_py, env, '--relocatable'])

    # Write the original deps (including metadata) as manifest.
    with open(manifest_path, 'wb') as f:
      f.write(repr(manifest) + '\n')

  # Create bin\python.bat on Windows to unify path where Python is found.
  if sys.platform.startswith('win'):
    bin_path = os.path.join(env, 'bin')
    if not os.path.isdir(bin_path):
      os.makedirs(bin_path)
    python_bat_path = os.path.join(bin_path, 'python.bat')
    if not os.path.isfile(python_bat_path):
      with open(python_bat_path, 'w') as python_bat_file:
        python_bat_file.write(PYTHON_BAT_WIN)

  if not quiet:
    print('Done creating environment')


def main(args):
  parser = argparse.ArgumentParser()
  parser.add_argument('--deps-file', '--deps_file', action='append',
                      help='Path to deps.pyl file (may be used multiple times, '
                      'default: bootstrap/deps.pyl)')
  parser.add_argument('-q', '--quiet', action='store_true', default=False,
                      help='Supress all output')
  parser.add_argument('env_path',
                      help='Path to place environment (default: %(default)s)',
                      default='ENV')
  opts = parser.parse_args(args)
  opts.deps_file = opts.deps_file or [os.path.join(ROOT, 'bootstrap/deps.pyl')]

  # Skip deps not available for this flavor of Python interpreter.
  #
  # Possible platform names:
  #   macosx_x86_64
  #   linux_i686
  #   linux_x86_64
  #   windows_i686
  #   windows_x86_64
  if sys.platform.startswith('linux'):
    osname = 'linux'
  elif sys.platform == 'darwin':
    osname = 'macosx'
  elif sys.platform == 'win32':
    osname = 'windows'
  else:
    osname = sys.platform
  if sys.maxsize == (2 ** 31) - 1:
    bitness = 'i686'
  else:
    bitness = 'x86_64'

  machine = platform.machine()
  arch = 'arm' if ('arm' in machine or 'aarch' in machine) else 'intel'
  if arch == 'arm':
    print('---------------------------')
    print('WARNING! WARNING! WARNING! ')
    print('---------------------------')
    print('The infra python environment is not available on ARM.')
    print('If you need to develop infra tools for ARM, use Go instead.')
    return

  plat = '%s_%s' % (osname, bitness)

  deps, kicked = filter_deps(merge_deps(opts.deps_file), plat)
  manifest = build_manifest(deps)
  activate_env(opts.env_path, manifest, opts.quiet)

  if not opts.quiet and kicked:
    print('---------------------------')
    print('WARNING! WARNING! WARNING! ')
    print('---------------------------')
    print('The following deps were skipped, they are not available on %s' %
          plat)
    for pkg, dep in sorted(kicked.iteritems()):
      print('  * %s (%s)' % (pkg, dep['version']))


if __name__ == '__main__':
  logging.basicConfig()
  LOGGER.setLevel(logging.DEBUG)
  sys.exit(main(sys.argv[1:]))