pip-pip

#!/usr/bin/env python
# -*- mode:python -*-

# $PIP_license: <Simplified BSD License>
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     Redistributions of source code must retain the above copyright notice,
#     this list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# $
# $RIKEN_copyright: Riken Center for Computational Sceience (R-CCS),
# System Software Development Team, 2020-2021
# $
# $PIP_PIP: Version 1.0.0$
#
# $Author: Atsushi Hori (R-CCS)
# Query:   procinproc-info@googlegroups.com
# User ML: procinproc-users@googlegroups.com
# $

"PiP-pip: PiP (Process-in-Process) package installation program"

from __future__ import print_function
import datetime
import time
import argparse
import shutil
import os
import sys
import subprocess as sp
import re

arch_supported   = [ 'x86_64', 'aarch64' ]
how_supported    = [ 'docker', 'spack', 'github', 'yum' ]
centos_supported = [ 7, 8 ]
pipver_supported = [ -1, 2, 3 ]

# not ready list: { how : [ ( arch, centos, pip ), ... ], ... }
not_ready = { 'yum' :    [ ( 'x86_64',  7, 1 ),
                           ( 'x86_64',  7, 2 ),
                           ( 'x86_64',  7, 3 ),
                           ( 'x86_64',  8, 1 ),
                           ( 'x86_64',  8, 2 ),
                           ( 'x86_64',  8, 3 ),
                           ( 'aarch64', 7, 1 ),
                           ( 'aarch64', 7, 2 ),
                           ( 'aarch64', 7, 3 ),
                           ( 'aarch64', 8, 1 ),
                           ( 'aarch64', 8, 2 ),
                           ( 'aarch64', 8, 3 ) ],

              'docker' : [ ( 'x86_64',  7, 1 ),
                           ( 'x86_64',  8, 1 ),
                           ( 'aarch64', 7, 1 ),
                           ( 'aarch64', 8, 1 ) ],

              'spack' :  [ ( 'x86_64',  7, 1 ),
                           ( 'x86_64',  8, 1 ),
                           ( 'aarch64', 7, 1 ),
                           ( 'aarch64', 8, 1 ) ],

              'github':  [ ( 'x86_64',  8, 1 ),
                           ( 'aarch64', 8, 1 ) ]
}

git_url_tab   = { 'github'   : 'https://github.com/RIKEN-SysSoft' }

git_repo_tab  = { 'github'   : { 'glibc': 'PiP-glibc.git',
                                 'pip'  : 'PiP.git',
                                 'gdb'  : 'PiP-gdb.git',
                                 'test' : 'PiP-Testsuite.git' } }

gnu_branch_tab = { 7 : ( 'centos/glibc-2.17-260.el7.pip.branch',
                         'centos/gdb-7.6.1-94.el7.pip.branch' ),
                   8 : ( 'centos/glibc-2.28-72.el8_1.1.pip.branch',
                         'centos/gdb-8.2-12.el8.pip.branch' ) }

docker_tagtab = { ( 'x86_64',  7, 2 ) : 'process-in-process/pip:2-centos7-amd64',
                  ( 'aarch64', 7, 2 ) : 'process-in-process/pip:2-centos7-arm64v8',
                  ( 'x86_64',  8, 2 ) : 'process-in-process/pip:2-centos8-amd64',
                  ( 'aarch64', 8, 2 ) : 'process-in-process/pip:2-centos8-arm64v8',
                  ( 'x86_64',  7, 3 ) : 'process-in-process/pip:3-centos7-amd64',
                  ( 'aarch64', 7, 3 ) : 'process-in-process/pip:3-centos7-arm64v8',
                  ( 'x86_64',  8, 3 ) : 'process-in-process/pip:3-centos8-amd64' ,
                  ( 'aarch64', 8, 3 ) : 'process-in-process/pip:3-centos8-arm64v8' }

def is_ready( how, arch, centos, pip_ver ):
    if ( arch, centos, pip_ver ) in not_ready[how]:
        return False
    return True

def resource_id( how, centos, pip_ver ):
    global ARGS, arch

    if how == 'yum':
        return 'process-in-process-v' + str( pip_ver )
    elif how == 'docker':
        try:
            return docker_tagtab[ arch, int(centos), pip_ver ]
        except:
            return None
    elif how == 'spack':
        return 'process-in-process@' + str( pip_ver )
    else:
        return None

def spack_url():
    spack_url     = 'https://github.com/spack/spack.git'
    spack_url_dev = 'git@git.sys.r-ccs.riken.jp:software/PIP-Spack.git'

    if how == 'spack':
        url = spack_url
    else:
        url = spack_url_dev
    bsnam = os.path.splitext( os.path.basename( url ) )[0]
    return url, bsnam

MSG_ENTER   = '>>>'
MSG_EXIT    = '<<<'
MSG_ERROR   = '<<E'
MSG_PREFIX  ='---'
MSG_PREFIX1 ='###'
ERR_PREFIX  ='PIP-PIP ERROR:'
WARN_PREFIX ='PIP-PIP WARNING:'
SEP = '\n=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n'

sudo_cmd      = []
query_prefix  = False
query_work    = False
spack_path    = None
DRYRUN        = False
FLAG_SUBDIR   = True
FLAG_NOUPDATE = False
FLAG_KEEP     = False
FLAG_QUIET    = False
FLAG_YES      = False
SUDO_ERR      = False
ERROR         = False

def mesg( words, newline, fs ):
    "general print message func"
    if words != []:
        for word in words[0:-1]:
            print( word, end=' ', file=fs )
        if newline:
            print( words[-1], file=fs )
        else:
            print( words[-1], end='', file=fs )
    elif newline:
        print( '', file=fs )
    fs.flush()

def message( words, newline=True, out=sys.stdout, log=None ):
    "print message"

    if out is not None and not FLAG_QUIET:
        mesg( words, newline, out )
    if log is not None and not DRYRUN:
        mesg( words, newline, log )

def err_msg( words, log=None ):
    "print error message"
    global ERROR

    ERROR = True
    message( [ ERR_PREFIX ] + words, out=sys.stderr, log=log )

def warn_msg( words, newline=True, log=None ):
    "print warning message"
    message( [ WARN_PREFIX ] + words, newline, out=sys.stderr, log=log )

urlparser = None
def concat_urls( url0, url1 ):
    "concatenate URLs"
    global urlparser

    if urlparser == None:
        try:
            import urllib.parse
            urlparser = urllib.parse
        except Exception as e0:
            try:
                import urlparse
                urlparser = urlparse
            except Exception as e1:
                err_msg( [ 'No urllib module:',   e0 ] )
                err_msg( [ 'No urlparse module:', e1 ] )
                NULL_FS.close()
                sys.exit( 1 )
    return urlparser.urljoin( url0+'/', url1 )

NULL_FS = open( os.devnull, 'w' )

def git_clone( dir, url, branch, log_fs ):
    if branch != '':
        if execute( dir, [ 'git', 'clone', '-b', branch, url ], log_fs ):
            return True
    else:
        if execute( dir, [ 'git', 'clone', url ], log_fs ):
            return True
    return False

def get_centos_version():
    "obtain RedHat/CentOS version"
    try:
        with open( '/etc/redhat-release', mode='r' ) as f:
            rhel = f.readline().split()
            while True:
                if rhel == []:
                    raise Exception( 'Unknown /etc/readhat-release content' )
                token = rhel.pop(0)
                if token == 'release':
                    verstr = rhel.pop(0)
                    ver = int( verstr.split( '.' ).pop( 0 ) )
                    return ver
    except Exception:
        err_msg( [ "Not a CentOS distribution" ] )
        sys.exit( 1 )

def cpu_arch():
    "obtain CPU architecture"
    return sp.check_output( [ 'uname', '-m' ] ).decode().split().pop(0)

def is_privileged( log_fs ):
    "check if the current user can sudo"
    global ARGS, sudo_cmd, SUDO_ERR

    if DRYRUN:
        return True

    if SUDO_ERR:
        return False

    user = os.environ['USER']
    if user == 'root':
        # no need of sudo
        sudo_cmd = []
        return True

    if sudo_cmd is not None:
        if sudo_cmd != []:
            return True

    sudo_rc = sp.call( [ '/bin/sh', '-c', 'sudo -S true < /dev/null' ],
                       stdout=NULL_FS, stderr=NULL_FS )
    if sudo_rc == 0:
        able_to_sudo = True
    else:
        able_to_sudo = False

    if ARGS.sudo:
        if able_to_sudo:
            sudo_cmd = [ 'sudo' ]
            return True
        sudo_cmd = []
        err_msg(  [ "'"+user+"'", 'is unable to sudo' ], log=log_fs )
        SUDO_ERR = True
    else:
        SUDO_ERR = True
        if able_to_sudo:
            err_msg(  [ '--sudo option is required' ], log=log_fs )
        else:
            err_msg(  [ "'"+user+"'", 'is unable to sudo' ], log=log_fs )
    return False

try:
    with open( '.pip-pip.cmd', mode='w' ) as f:
        nowstr = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S' )
        print( '#', nowstr, file=f )
        for tkn in sys.argv:
            print( tkn, end='', file=f )
        print( '', file=f )
except:
    pass

cmd = sys.argv[0]

parser = argparse.ArgumentParser( description='pip-pip: Process-in-Process '+
                                  'package installing program',
                                  prog='pip-pip',
                                  add_help = True )
parser.add_argument( '--how', '-H',
                     help    = 'specifying how to install ' \
                         '[yum|docker|spack|GITHUB]',
                     type    = str,
                     action  = 'append' )
parser.add_argument( '--version', '-v',
                     help    = "specifying PiP version (2 or 3. default '2')",
                     type    = str,
                     action  = 'append' )
parser.add_argument( '--centos', '-c',
                     help    = "Docker only: specifying RedHat/CentOS version (7 or 8. default 8)",
                     type    = str,
                     action  = 'append' )
parser.add_argument( '--prefix', '-p',
                     help    = "install (prefix) directory " \
                        "(github [, spack]. default './install')",
                     type    = str,
                     default = 'install',
                     nargs   = '?' )
parser.add_argument( '--work', '-w',
                     help    = "working directory (default './work')",
                     type    = str,
                     default = 'work',
                     nargs   = '?' )
parser.add_argument( '--keep', '-k',
                     help    = 'keep working directory. ' +
                     'Work directory(ies) is (are) deleted when succeeded by default',
                     action  = 'store_true' )
parser.add_argument( '--sudo', '-s',
                     help    = 'allow sudo (docker)',
                     action  = 'store_true' )
parser.add_argument( '--yes', '-y',
                     help    = "answer 'yes' to all questions",
                     action  = 'store_true' )
parser.add_argument( '--quiet', '-q',
                     help    = 'quiet mode',
                     action  = 'store_true' )
parser.add_argument( '--noglibc',
                     help    = 'do not build PiP-glibc and PiP_gdb (github)',
                     action  = 'store_true' )
parser.add_argument( '--nogdb',
                     help    = 'do not build PiP-gdb (github)',
                     action  = 'store_true' )
parser.add_argument( '--notest',
                     help    = 'do not test installed PiP (and PiP-gdb)',
                     action  = 'store_true' )
parser.add_argument( '--noupdate',
                     help    = 'do not update if specified PiP installation exists',
                     action  = 'store_true' )
parser.add_argument( '--nosubdir',
                     help    = 'do not create subdirectory(ies) in the prefix directory',
                     action  = 'store_true' )
parser.add_argument( '--threshold', '-t',
                     help    = 'threshold of the number of PiP Testsuite errors',
                     type    = int,
                     nargs   = '?',
                     default = 10 )
parser.add_argument( '--ready', '-r',
                     help    = "list ready to install and exit",
                     type    = str,
                     nargs   = '?',
                     default = '' )
parser.add_argument( '--dryrun', '--dry', '-d',
                     help    = 'dry run',
                     action  = 'store_true' )
ARGS = parser.parse_args()

DRYRUN        = ARGS.dryrun
FLAG_QUIET    = ARGS.quiet
FLAG_NOUPDATE = ARGS.noupdate
FLAG_KEEP     = ARGS.keep
FLAG_YES      = ARGS.yes

if ARGS.threshold < 0:
    threshold = [ 'PIP_TEST_THRESHOLD=10' ]
else:
    threshold = [ 'PIP_TEST_THRESHOLD='+str(ARGS.threshold) ]

if ARGS.nosubdir:
    FLAG_SUBDIR = False

def check_command( command ):
    "check if the Linux command exists"

    if DRYRUN:
        return True

    try:
        if sp.check_call( ['sh','-c','type '+command+' >/dev/null 2>&1' ] ) == 0:
            return True
    except:
        return False
    return False

def expand( lst, lower=False ):
    "flatten list"
    if lst == [] or lst is None:
        return []
    if lower:
        token = lst.pop( 0 ).lower()
    else:
        token = lst.pop( 0 )
    if ',' in token:
        return token.split( ',' ) + expand( lst )
    if '+' in token:
        return token.split( '+' ) + expand( lst )
    return [ token ] + expand( lst )

## check how
how_list = expand( ARGS.how, lower=True )
how_all = how_supported

os_list = expand( ARGS.centos )
if os_list == []:
    os_list = [ '8' ]
centos_list = []

if 'all' in how_list or 'All' in how_list:
    how_list = how_all
elif how_list == []:
    how_list = [ 'github' ]
else:
    errlist = []
    howlist = []
    for h in how_list:
        if not h in how_all:
            errlist.append( h )
    if errlist != []:
        if len(errlist) == 1:
            err_msg( [ errlist[0], 'is not supported value of --how' ]  )
        else:
            err_msg( [ ','.join(errlist), 'are not supported values of --how' ]  )
        parser.print_help()
    else:
        for h in how_all:
            if h in how_list:
                howlist.append( h )
    if howlist == []:
        ERROR = True
    how_list = howlist

if len( how_list ) > 1 and not FLAG_SUBDIR:
    err_msg( [ 'Multiple versions cannot be installed without having sub-directoris' ]  )

if 'yum' in how_list:
    if not check_command( 'yum' ):
        warn_msg( [ "'yum' command not found -- yum installation skipped" ] )
        how_list.remove( 'yum' )
    elif not is_privileged( None ):
        how_list.remove( 'yum' )
if 'docker' in how_list:
    if not check_command( 'docker' ):
        warn_msg( [ "'docker' command not found -- docker installation skipped" ] )
        how_list.remove( 'docker' )
    elif not is_privileged( None ):
        how_list.remove( 'docker' )
    elif 'all' in os_list or 'ALL' in os_list:
        centos_list = [ '7', '8' ]
    else:
        for centos in os_list:
            if not centos in [ '7', '8' ]:
                warn_msg( [ "Unsupported CentOS version (" + centos + ")" ] )
            elif not centos in centos_list:
                centos_list.append( centos )
        if centos_list is []:
            warn_msg( [ "Docker installation skipped" ] )
            how_list.remove( 'docker' )
if 'spack' in how_list:
    if check_command( 'spack' ):
        spack_path = 'spack'
    elif not check_command( 'git' ):
        spack_path = ''
        warn_msg( [ "'spack' and 'git' commands not found -- spack installation skipped" ] )
        how_list.remove( 'spack' )
if 'github' in how_list:
    if not check_command( 'git' ):
        err_msg( [ "'git' command not found -- git installation skipped" ] )
        if 'github' in how_list:
            how_list.remove( 'github' )

if how_list == []:
    ERROR = True

## check Linux distribution and version
centos_version = get_centos_version()
if not centos_version in centos_supported:
    err_msg( [ 'RedHat/CentOS-'+str(centos_version), 'is not supported' ] )
    sys.exit( 1 )

## Check PiP version
version = expand( ARGS.version )
pip_vers = []
if 'ALL' in version:
    if centos_version == 7:
        pip_vers = [ 1, 2, 3 ]
    else:
        pip_vers = [ 2, 3 ]
elif 'all' in version:
    pip_vers = [ 2, 3 ]
elif version == []:
    pip_vers = [ 2 ]
else:
    version = sorted( set( version ) )
    errlist = []
    for ver_str in version:
        try:
            ver = int( ver_str )
            if ver in pipver_supported:
                pip_vers.append( abs( ver ) )
            else:
                errlist.append( ver_str )
        except:
            errlist.append( ver_str )
    errlen = len( errlist )
    if errlen > 0:
        if errlen == 1:
            err_msg( [ errlist[0], 'is not an supported value of --version' ] )
        else:
            err_msg( [ ','.join(errlist),
                       'are not supported values of --version' ] )

# check arch
arch = cpu_arch()
if not arch in arch_supported:
    err_msg( [ arch, 'is not supported by PiP' ] )

if ARGS.ready != '':
    list_ready = []
    if ARGS.ready == 'all':
        for how in how_supported:
            if how in [ 'yum', 'docker' ] and SUDO_ERR:
                continue
            for cpu in arch_supported:
                for linux in [ 7, 8 ]:
                    for pip in [ 2, 3 ]:
                        if is_ready( how, cpu, linux, pip ):
                            list_ready.append( ( how, cpu, linux, pip ) )
    elif ARGS.ready == 'ALL':
        for how in how_supported:
            if how in [ 'yum', 'docker' ] and SUDO_ERR:
                continue
            for cpu in arch_supported:
                for linux in [ 7, 8 ]:
                    for pip in [ 1, 2, 3 ]:
                        if is_ready( how, cpu, linux, pip ):
                            list_ready.append( ( how, cpu, linux, pip ) )
    elif ARGS.ready in [ 'arch', 'cpu' ]:
        for how in how_supported:
            if how in [ 'yum', 'docker' ] and SUDO_ERR:
                continue
            for linux in [ 7, 8 ]:
                for pip in [ 1, 2, 3 ]:
                    if is_ready( how, arch, linux, pip ):
                        list_ready.append( ( how, arch, linux, pip ) )
    elif ARGS.ready in [ 'linux', 'redhat', 'centos' ]:
        for how in how_supported:
            if how in [ 'yum', 'docker' ] and SUDO_ERR:
                continue
            for arch in arch_supported:
                for pip in [ 1, 2, 3 ]:
                    if is_ready( how, arch, centos_version, pip ):
                        list_ready.append( ( how, arch, centos_version, pip ) )
    else:
        for how in how_list:
            if how in [ 'yum', 'docker' ] and SUDO_ERR:
                continue
            for pip in pip_vers:
                if is_ready( how, arch, centos_version, pip ):
                    list_ready.append( ( how, arch, centos_version, pip ) )
    if list_ready == []:
        message( [ 'Nothing ready' ] )
    else:
        message( [ 'Ready list' ] )
        for ( how, arch, linux, pip ) in list_ready:
            msgstr = '{0:>10}\t{1:>10}\tRedHat/CentOS:{2:}    \tPiP-{3:}'.format(how,arch,linux,pip)
            message( [ msgstr ] )
    sys.exit( 0 )

def make_directory( path ):
    "make directory"

    if DRYRUN:
        return path

    try:
        if not os.path.isdir( path ):
            os.mkdir( path )
        return path
    except Exception as e:
        err_msg( [ 'make_directory:', e ] )
        return None

def query( prompt, ans ):
    try:
        while True:             # python2
            choice = raw_input( prompt ).lower()
            if choice != '' and choice[0] in ans:
                return choice[0]
            continue
    except:
        try:
            while True:         # python3
                choice = input( prompt ).lower()
                if choice != '' and choice[0] in ans:
                    return choice[0]
                continue
        except:
            message( [] )
            NULL_FS.close()
            sys.exit( 1 )

# check and create prefix dir
def check_prefix( pref ):
    "check the prefix directory"
    global query_prefix, ERROR

    if DRYRUN:
        return True
    if os.path.isfile( pref ):
        err_msg( [ pref, 'is not a directory but a file' ] )
        return False
    if os.path.isdir( pref ):
        if not FLAG_YES and not query_prefix:
            warn_msg( [ 'Prefix directory already exists: ', pref ] )
            ans = query( 'Are you sure to install into the prefix directory? ' \
                         "'yes', 'all' or 'no' [Y/A/N]: ",
                         'yan' )
            if ans == 'n':
                return False
            if ans == 'a':
                query_prefix = True # never ask again
    return True

def prefix_path( prefix, how, ver ):
    "make the prefix path"
    global ERROR

    if not FLAG_SUBDIR:
        return prefix
    return os.path.join( prefix,
                         arch + '_' +
                         'cetos-' + str( centos_version ) + '_' +
                         how + '_' +
                         'pip-' + str( ver ) )

def create_prefix( prefix, how, ver ):
    "create the prefix directory"

    pdir = prefix_path( prefix, how, ver )
    if DRYRUN:
        return pdir
    try:
        if not os.path.isdir( pdir ):
            os.makedirs( pdir )
        return pdir
    except Exception as e:
        err_msg( [ 'create_prefix:', e ] )
        return None

def delete_work_dir( wdir ):
    "delete work directory"

    if DRYRUN:
        message( [ '[DRY] Deleting work directory ('+wdir+') .. done' ] )
        return True

    message( [ 'Deleting work directory ('+wdir+') ..' ], newline=False )
    try_shutil  = True
    retry_count = 3
    retry       = retry_count
    retry_wait  = 1
    while True:
        try:
            if try_shutil:
                try_shutil = False
                shutil.rmtree( wdir )
            else:
                if sp.run( [ '/bin/sh', '-c', 'rm -f -r ' + wdir ],
                           check=False,
                           stdout=NULL_FS, stderr=NULL_FS ).returncode != 0:
                    raise Exception( 'rm -r -f ' + wdir + ' FAILD' )
                # success
                message( [ ' done' ] )
                return True

        except:
            if retry == 0:
                message( [ 'gave up' ] )
                return False
            message( [ 'retrying.. ' ], newline=False )
            time.sleep( retry_wait )
            retry_wait *= 2
            retry -= 1
            continue

# check and create work dir
def work_path( work, how, ver ):
    "create work directory path"
    path = os.path.join( work,
                         arch + '_' +
                         'centos-' + str( centos_version ) + '_' +
                         how + '_' +
                         'pip-' + str( ver ) )
    return path

def check_work( work ):
    "check working directory"
    global list_del_work, query_work, ERROR

    if DRYRUN:
        return True
    if os.path.isfile( work ):
        err_msg( [ work, 'is not a directory but a file' ] )
        return False
    if os.path.isdir( work ):
        if not FLAG_YES and not query_work:
            warn_msg( [ 'Work directory already exists:', work ] )
            ans = query( 'Delete work directory before install? ' \
                         "'yes', 'all' or 'no' [Y/A/N]: ",
                         'yan' )
            if ans == 'n':
                ERROR = True
                return False
            if ans == 'a':
                query_work = True # never ask again
        list_del_work.append( work )
    return True

def create_work( work ):
    "make work directory"

    if DRYRUN:
        return work, None
    try:
        log_dir = os.path.join( work, 'log' )
        if not os.path.isdir( log_dir ):
            os.makedirs( log_dir )
        log_path = os.path.join( log_dir, 'pip-pip.log' )
    except Exception as e:
        err_msg( [ 'create_work:', e ] )
        return None, None
    return work, log_path

def check_file( file ):
    "check if the file exists or not"

    if DRYRUN:
        return True
    return os.path.isfile( file )

def do_poll( proc, t, dot, inc, tmax, log_fs, silent ):
    "do poll and output dots"

    t0 = t
    if t < 100:
        delta = 0.2
    elif t < 1000:
        delta = 1.0
    else:
        delta = 10.0

    td = 0.0
    while True:
        ret = proc.poll()
        if ret is not None or t >= tmax:
            if t < 60:
                tu = t
                unit = 's'
            elif t < 3600:
                tu = t / 60
                unit = 'm'
            else:
                tu = t / 3600
                unit = 'h'

            if ret is None:
                if not silent:
                    message( [ str(int(tu))+unit ], newline=False )
                return None, t
            if not silent:
                message( [ ' '+str(int(tu))+unit ], newline=False )
            break
        if td >= inc:
            t += inc
            td = 0.0
            if not silent:
                message( [ dot ], newline=False )

        time.sleep( delta )
        td += delta

    if ret != 0:
        if not silent:
            message( [ ' - NG' ], log=log_fs )
        return False, t
    if not silent:
        message( [ ' - OK' ], log=log_fs )
    return True, t

def execute( wd, cmd, log_fs, silent=False, env=[] ):
    "execute a Linux command"
    global ERROR

    if DRYRUN:
        if not silent:
            message( [ '[DRY]' ] + cmd, log=log_fs )
        return True
    if ERROR:
        if not silent:
            message( [ MSG_PREFIX ] +
                     cmd +
                     [ '-- execution skipped due to prev. error' ],
                     log=log_fs )
        return False

    try:
        if not silent:
            message( [ MSG_PREFIX ] + env + cmd, newline=False, log=log_fs )
            message( [], newline=True, log=log_fs, out=None )
        if wd is not None:
            cmd_list = [ 'cd', wd, '&&' ] + cmd
        else:
            cmd_list = cmd
        shell_cmd = [ '/bin/sh', '-c', ' '.join( env + cmd_list ) ]
        proc = sp.Popen( shell_cmd, close_fds=True,
                         stdout=log_fs, stderr=log_fs )

        if not silent:
            message( ' ', newline=False )
        inctab = [ ( '.',       1,       10 ),
                   ( '.',       6,       60 ),
                   ( '.',      60,      360 ),
                   ( '.',     360,     3600 ),
                   ( '.',    3600,  24*3600 ),
                   ( '.', 24*3600, 9999*999 ) ]
        t = 0
        for ( dot, inc, tmax ) in inctab:
            ( ret, t ) = do_poll( proc, t, dot, inc, tmax, log_fs, silent )
            if ret is not None:
                time.sleep( 1 )     # wait for stdout/stderr flushing
                return ret
    except Exception as e:
        err_msg( [ '\nException:', e ], log=log_fs )
        return False

def is_already_installed( prefix, how, pip_ver, centos ):
    global spack_path

    if DRYRUN:
        return False

    if how == 'yum':
        yum_pip = resource_id( how, centos, pip_ver )
        if execute( None, [ 'yum', 'list', 'installed', yum_pip ], NULL_FS, True ):
            return True
    elif how == 'docker':
        docker_pip = resource_id( how, centos, pip_ver )
        if execute( None,
                    sudo_cmd + [ 'docker', 'inspect', docker_pip ],
                    NULL_FS, True ):
            return True;
    elif how == 'spack' and spack_path is not None:
        spack_pip = resource_id( how, centos, pip_ver )
        if execute( None, [ spack_path, 'find', spack_pip ], NULL_FS, True ):
            return True;
    else: # GIT
        libpipso = os.path.join( prefix, 'lib', 'libpip.so' )
        if os.path.isfile( libpipso ):
            return True
    return False

def install_yum( pip_ver, work_dir, log_fs ):
    global threshold
    "yum install"
    yum_pip = resource_id( 'yum', centos_version, pip_ver )
    message( [ MSG_ENTER, 'Yum install', yum_pip ], log=log_fs )
    retuple = ( 'yum', yum_pip, '' )
    if not execute( None,
                    sudo_cmd + [ 'yum', 'reinstall', '-y', yum_pip ],
                    log_fs ):
        return False, retuple

    if not ARGS.notest:
        repo_test  = concat_urls( git_url_tab['github'],
                                  git_repo_tab['github']['test']  )

        message( [ MSG_PREFIX1, 'Testing', repo_test ], log=log_fs )
        if not git_clone( work_dir, repo_test, '', log_fs ):
            return False, retuple
        test_dir = os.path.join( work_dir, 'PiP-Testsuite' )
        configure = os.path.join( test_dir, 'configure' )
        if not execute( test_dir,
                        [ configure, '--with-pip=/usr' ],
                        log_fs ):
            return False, retuple
        if not execute( test_dir, [ 'make', 'test' ], log_fs, env=threshold ):
            return False, retuple
    return True, retuple

def install_docker( pip_ver, centos, log_fs ):
    "docker install"
    docker_pip = resource_id( 'docker', centos, pip_ver )
    message( [ MSG_ENTER, 'Docker install', docker_pip ], log=log_fs )
    retuple = ( 'docker', docker_pip, '' )
    if execute( None,
                sudo_cmd + [ 'docker', 'inspect', docker_pip ], log_fs ):
        message( [ MSG_PREFIX1,
                   'Removing existing image', docker_pip ],
                 log=log_fs )
        if not execute( None,
                        sudo_cmd + [ 'docker', 'rmi', docker_pip ],
                        log_fs ):
            err_msg( [ 'Failed to remove existing image', docker_pip ],
                       log=log_fs )
            return False, retuple
    message( [ MSG_PREFIX1, 'Pulling existing image', docker_pip ], log=log_fs )
    if not execute( None, sudo_cmd + [ 'docker', 'pull', docker_pip ], log_fs ):
        return False, retuple
    return True, retuple

def install_spack( prefix, how, pip_ver, log_fs ):
    "Spack install"
    global spack_path

    spack_pip = resource_id( 'spack', centos_version, pip_ver )
    message( [ MSG_ENTER, 'Spack install', spack_pip ], log=log_fs )
    retuple = ( 'spack', spack_pip, '' )

    if spack_path == '':        # Spack not found
        return False, retuple

    if spack_path is None:
        prefix_dir = create_prefix( prefix, how, pip_ver )
        if prefix_dir is None:
            return False, retuple
        ( url, bsnam ) = spack_url()
        spack_dir = os.path.join( prefix_dir, bsnam )
        if os.path.isdir( os.path.join( spack_dir, '.git' ) ):
            message( [ MSG_PREFIX1, 'Pulling Spack into', prefix_dir ], log=log_fs )
            if not execute( spack_dir, [ 'git', 'pull', url ], log_fs ):
                return False, retuple
        else:
            message( [ MSG_PREFIX1, 'Cloning Spack at', prefix_dir ], log=log_fs )
            if not git_clone( prefix_dir, url, '', log_fs ):
                spack_path = ''     # mark as not found
                return False, retuple
        spack_path = os.path.join( prefix_dir, bsnam, 'bin', 'spack' )

    retuple = ( 'spack', spack_pip, spack_path )
    execute( None, [ spack_path, 'compiler', 'find' ], log_fs )
    if execute( None, [ spack_path, 'find', spack_pip ], log_fs ):
        message( [ MSG_PREFIX1, 'Uninstalling', spack_pip ], log=log_fs )
        if not execute( None, [ spack_path, 'uninstall', '-y', spack_pip ], log_fs ):
            return False, retuple
    message( [ MSG_PREFIX1, 'Installing', spack_pip ], log=log_fs )
    if not execute( None, [ spack_path, 'install',  spack_pip ], log_fs ):
        return False, retuple
    return True, retuple

def byte_to_char( byte_str ):
    "convert byte string to char (for compatibility between Python2 and 3)"
    try:
        if byte_str == '' or byte_str[0] in 'abc':
            ch_str = byte_str   # python2
        else:
            ch_str = byte_str
    except:                     # python3
        ch_str = byte_str.decode()
    return ch_str

def wget_install( work_dir, prefix_dir, pkgnam_b, url_b, log_fs ):
    "wget install for required package"
    pkgname = byte_to_char( pkgnam_b )
    url     = byte_to_char( url_b    )
    if not execute( work_dir, [ 'wget', url ], log_fs ):
        err_msg( [ 'wget_install: wget', url, '-- FAILED' ], log=log_fs )
        raise Exception()
    [ mod_tar ] = re.findall( '.*/(.*)', url )
    if not check_file( os.path.join( work_dir, mod_tar ) ):
        err_msg( [ 'wget_install:', mod_tar, '-- unable to find' ], log=log_fs )
        raise Exception()
    if not execute( work_dir, [ 'tar', 'xzf', mod_tar ], log_fs ):
        err_msg( [ 'wget_install: tar xzf', mod_tar, '-- FAILED' ], log=log_fs )
        raise Exception()
    os.remove( os.path.join( work_dir, mod_tar ) )
    [ mod_name ] = re.findall( r'.*/(.*)\.tar.gz', url )
    mod_work = os.path.join( work_dir, mod_name )
    if not execute( mod_work,
                    [ './configure', '--prefix=' + prefix_dir ],
                    log_fs ):
        err_msg( [ 'wget_install: '
                   './configure', '--prefix='+prefix_dir,
                   '['+mod_name+'] -- FAILED' ],
                   log=log_fs )
        raise Exception()
    if not execute( mod_work, [ 'make' ], log_fs ):
        err_msg( [ 'wget_install: make ['+mod_name+'] -- FAILED' ],
                   log=log_fs )
        raise Exception()
    if not execute( mod_work, [ 'make', 'install' ], log_fs ):
        err_msg( [ 'wget_install: make install ['+mod_name+'] -- FAILED' ],
                   log=log_fs)
        raise Exception()
    return '--package=' + pkgname + ':' + prefix_dir

def install_git( prefix, repo, pip_ver, work_dir, log_fs  ):
    "Git (source) install"
    global threshold

    branch_pip = 'pip-' + str( pip_ver )

    retuple = ( 'git', repo + '@' + branch_pip, '' )
    if repo == 'github':
        git_repo = git_url_tab[repo]
        pip = 'PiP'
    else:
        err_msg( [ 'Unknown git repo:', repo ], log=log_fs )
        return False, retuple

    message( [ MSG_ENTER, 'Git install', git_repo, 'PiP-v'+str( pip_ver ) ],
             log=log_fs )

    repo_glibc = concat_urls( git_repo, git_repo_tab[repo]['glibc'] )
    repo_pip   = concat_urls( git_repo, git_repo_tab[repo]['pip']   )
    repo_gdb   = concat_urls( git_repo, git_repo_tab[repo]['gdb']   )
    repo_test  = concat_urls( git_repo, git_repo_tab[repo]['test']  )
    git_pip = repo_pip + '@' + branch_pip
    retuple = ( 'git', git_pip, '' )
    if centos_version == 7:
        ( branch_glibc, branch_gdb ) = gnu_branch_tab[7]
    elif centos_version == 8:
        ( branch_glibc, branch_gdb ) = gnu_branch_tab[8]
    else:
        err_msg( [ 'Unsupported RedHat/CentOS version:',
                   str( centos_version ) ],
                 log=log_fs )
        return False, retuple

    prefix_dir = create_prefix( prefix, repo, pip_ver )
    if prefix_dir is None:
        return False, retuple
    retuple = ( 'git', git_pip, prefix_dir )

    glibc_srcdir  = os.path.join( work_dir,     pip + '-glibc'  )
    build_script  = os.path.join( glibc_srcdir, 'build.sh'      )
    glibc_build   = os.path.join( work_dir,     'glibc-build'   )
    glibc_install = os.path.join( prefix_dir,   'glibc-install' )
    if ARGS.noglibc:
        message( [ MSG_PREFIX1, 'Skipping PiP-glibc build (--noglibc)' ],
                 log=log_fs )
    else:
        message( [ MSG_PREFIX1, 'Cloning', repo_glibc, branch_glibc ],
                 log=log_fs )
        if not git_clone( work_dir, repo_glibc, branch_glibc, log_fs ):
            return False, retuple
        message( [ MSG_PREFIX1, 'Building', repo_glibc ], log=log_fs )
        if make_directory( glibc_build ) is None:
            return False, retuple
        if make_directory( glibc_install ) is None:
            return False, retuple
        if not execute( glibc_build, [ build_script, glibc_install ], log_fs ):
            return False, retuple
        piplnlibs = os.path.join( glibc_install, 'bin', 'piplnlibs' )
        if not check_file( piplnlibs ):
            err_msg( [ 'Unable to find: ', piplnlibs ], log=log_fs )
            return False, retuple

    message( [ MSG_PREFIX1, 'Cloning', repo_pip, branch_pip ], log=log_fs )
    if not git_clone( work_dir, repo_pip, branch_pip, log_fs ):
        return False, retuple

    message( [ MSG_PREFIX1, 'Building', repo_pip ], log=log_fs )
    pip_dir   = os.path.join( work_dir, pip )
    configure = os.path.join( pip_dir,  'configure' )
    if ARGS.noglibc:
        if not execute( pip_dir,
                        [ configure, '--prefix=' + prefix_dir ],
                        log_fs ):
            return False, retuple
    else:
        if not execute( pip_dir,
                        [ configure,
                          '--prefix=' + prefix_dir,
                          '--with-glibc-libdir=' + glibc_install + '/lib' ],
                        log_fs ):
            return False, retuple
    if not execute( pip_dir, [ 'make' ], log_fs ):
        return False, retuple
    if not execute( pip_dir, [ 'make install' ], log_fs ):
        return False, retuple
    if not execute( pip_dir, [ 'make', 'doc-install' ], log_fs ):
        return False, retuple

    if not ARGS.notest:
        message( [ MSG_PREFIX1, 'Testing', repo_pip ], log=log_fs )
        if not git_clone( work_dir, repo_test, '', log_fs ):
            return False, retuple
        test_dir = os.path.join( work_dir, pip + '-Testsuite' )
        configure = os.path.join( test_dir, 'configure' )
        if not execute( test_dir,
                        [ configure, '--with-pip=' + prefix_dir ],
                        log_fs ):
            return False, retuple
        if not execute( test_dir, [ 'make', 'test' ], log_fs, env=threshold ):
            return False, retuple

    if ARGS.noglibc or ARGS.nogdb:
        message( [ MSG_PREFIX1,
                   'Skipping PiP-gdb build (--noglibc or --nogdb)' ],
                 log=log_fs )
    else:
        message( [ MSG_PREFIX1, 'Cloning', repo_gdb, branch_gdb ],
                 log=log_fs )
        if not git_clone( work_dir, repo_gdb, branch_gdb, log_fs ):
            return False, retuple

        inst_pkgs = []
        if not DRYRUN:
            message( [ MSG_PREFIX1,
                       'Checking required packages', repo_gdb ],
                     log=log_fs )
            gdb_dir = os.path.join( work_dir, pip + '-gdb' )
            build_script = os.path.join( gdb_dir, 'build.sh' )
            try:
                psproc = sp.Popen( [ build_script, '--missing' ],
                                   stdout=sp.PIPE, stderr=log_fs )
                while True:
                    mod = psproc.stdout.readline()
                    if mod == '':
                        break
                    missing = mod.split()
                    if missing == []:
                        break
                    [ pkgname, url ] = missing
                    installed = wget_install( work_dir,
                                              prefix_dir,
                                              pkgname,
                                              url,
                                              log_fs )
                    if not installed in inst_pkgs:
                        inst_pkgs += [ installed ]
            except Exception as e:
                err_msg( [ build_script+' --missing: FAILED', e ],
                         log=log_fs )
                return False, retuple

        message( [ MSG_PREFIX1, 'Building', repo_gdb ], log=log_fs )
        gdb_dir = os.path.join( work_dir, pip + '-gdb' )
        build_script = os.path.join( gdb_dir, 'build.sh' )
        if not execute( gdb_dir,
                        [ build_script,
                          '--prefix='   + prefix_dir,
                          '--with-pip=' + prefix_dir ] + inst_pkgs,
                        log_fs ):
            return False, retuple
        pip_gdb = os.path.join( prefix_dir, 'bin', 'pip-gdb' )
        if not check_file( pip_gdb ):
            err_msg( [ 'Unable to find: ', pip_gdb ], log=log_fs )
            return False, retuple

        if not ARGS.notest:
            message( [ MSG_PREFIX1, 'Testing', repo_gdb ], log=log_fs )
            test_script = os.path.join( gdb_dir, 'test.sh' )
            if not execute( gdb_dir, [ test_script ], log_fs ):
                return False, retuple
    if not FLAG_KEEP:
        delete_work_dir( work_dir )
    return True, retuple

if ERROR:
    NULL_FS.close()
    sys.exit( 1 )

prefix_dir = os.path.realpath( os.path.expanduser( ARGS.prefix ) )
work_dir   = os.path.realpath( os.path.expanduser( ARGS.work   ) )

# check if work dir can be deleted
common = os.path.commonprefix( [ work_dir, prefix_dir ] )
if common is work_dir:          # this means work_dir includes prefix_dir
    if not FLAG_KEEP:
        err_msg( [ 'Work dir includes prefix dir and ' +
                   'work dir cannot be deleted.' ] )

if ERROR:
    NULL_FS.close()
    sys.exit( 1 )

list_ready     = []
list_not_ready = []
list_install   = []
list_pass0     = []
list_pass1     = []
list_skipped   = []
list_noupdate  = []

if FLAG_NOUPDATE:
    message( [] )
    message( [ 'Checking if already installed (--noupdate)' ] )

for ver in pip_vers:
    for how in how_list:
        if how is 'docker':
            for centos in centos_list:
                list_install.append( ( how, ver, centos ) )
        else:
            list_install.append( ( how, ver, centos_version ) )

# check if the specified installatios are ready or not

for install in list_install:
    ( how, ver, centos ) = install
    if how is 'docker':
        if not is_ready( how, arch, centos, ver ):
            list_not_ready.append( ( how, ver, centos ) )
        elif is_already_installed( '', how, ver, centos ) and FLAG_NOUPDATE:
            list_noupdate.append( ( how, ver, centos ) )
        else:
            list_pass0.append( ( how, ver, centos ) )
    else:
        prefdir = prefix_path( prefix_dir, how, ver )
        if not is_ready( how, arch, centos, ver ) :
            list_not_ready.append( ( how, ver, centos_version ) )
        elif is_already_installed( prefdir, how, ver, centos_version ) and FLAG_NOUPDATE:
            list_noupdate.append( ( how, ver, centos_version ) )
        else:
            list_pass0.append( ( how, ver, centos_version ) )

# check the prefix (install) directories

for ( how, ver, centos ) in list_pass0:
    if check_prefix( prefix_path( prefix_dir, how, ver ) ):
        list_pass1.append( ( how, ver, centos ) )
    else:
        list_skipped.append( ( how, ver, centos ) )

# check work dir, delete if exists

list_del_work = []
for ( how, ver, centos ) in list_pass1:
    # list_del_work may be updated in check_work()
    if check_work( work_path( work_dir, how, ver ) ):
        list_ready.append( ( how, ver, centos ) )
    else:
        list_skipped.append( ( how, ver, centos ) )

ERROR = False

### NOW we can actually do the installation(s)

message( [] )
message( [ 'RedHat/CentOS:\t\t', centos_version ] )
message( [ 'CPU Architecture:\t', arch ])

if list_ready != []:
    message( [] )
    message( [ 'List of installations' ] )
    for ( how, ver, centos ) in list_ready:
        message( [ how.upper(), '\tPiP-v' + str( ver ) ] )
        prefdir = prefix_path( prefix_dir, how, ver )
        if how == 'spack':
            message( [ '  Prefix dir:\t', prefdir ] )
        elif how == 'docker':
            message( [ '  Docker Tag:\t', resource_id( how, centos, ver ) ] )
        elif how == 'github':
            message( [ '  Prefix dir:\t', prefdir ] )
            message( [ '  Work dir:\t', work_path( work_dir, how, ver ) ] )

if list_skipped != []:
    message( [] )
    if len( list_skipped ) == 1:
        warn_msg( [ 'The following is skipped' ] )
    else:
        warn_msg( [ 'The followings are skipped' ] )

    for ( how, ver ) in list_skipped:
        message( [ '\t'+how.upper()+': PiP-v' + str( ver ) ] )

if list_not_ready != []:
    message( [] )
    if len( list_not_ready ) == 1:
        warn_msg( [ 'The following is not ready and',
                    'installtion will be skipped' ] )
    else:
        warn_msg( [ 'The followings are not ready and',
                    'installtion will be skipped' ] )

    for ( how, ver, centos ) in list_not_ready:
        message( [ '\t'+how.upper()+': PiP-v' + str( ver ) ] )

if list_noupdate != []:
    message( [] )
    if len( list_noupdate ) == 1:
        warn_msg( [ 'The following is already installed and',
                    'installtion will be skipped (--noupdate)' ] )
    else:
        warn_msg( [ 'The followings are already installed and',
                    'installtion will be skipped (--noupdate)' ] )

    for ( how, ver ) in list_noupdate:
        message( [ '\t'+how.upper()+': PiP-v' + str( ver ) ] )

if list_del_work != []:
    message( [] )
    if len( list_del_work ) == 1:
        warn_msg( [ 'The following directory will be deleted' ] )
    else:
        warn_msg( [ 'The following directories will be deleted' ] )
    for wrk in list_del_work:
        message( [ '\t'+wrk ] )

if list_ready == []:
    err_msg( [ 'Nothing to install' ] )
    NULL_FS.close()
    sys.exit( 1 )

if not FLAG_YES and not DRYRUN:
    ans = query( '\nProceed? [Y/N]: ', 'yn' )
    if ans == 'n':
        NULL_FS.close()
        sys.exit( 1 )

## let's start installation(s)

if list_del_work != []:
    message( [] )
    for wrk in list_del_work:
        delete_work_dir( wrk )

ERROR = False

list_success  = []
list_failure  = []
list_skipped  = []
list_docker_images = []

for ( how, pip_ver, centos ) in list_ready:
    ERROR = False
    message( [] )

    ( wdir, log_path ) = create_work( work_path( work_dir, how, pip_ver ) )
    if wdir is None:
        NULL_FS.close()
        sys.exit( 1 )

    try:
        log_fs = None
        if not DRYRUN:
            log_fs = open( log_path, mode='w' )

        now_str = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S' )
        message( [ now_str, SEP ], log=log_fs, out=None )

        if how == 'yum':
            retuple = install_yum( pip_ver, wdir, log_fs )
        elif how == 'docker':
            retuple = install_docker( pip_ver, centos, log_fs )
        elif how == 'spack':
            retuple = install_spack( prefix_dir, how, pip_ver, log_fs )
        elif how == 'github':
            retuple = install_git( prefix_dir, how, pip_ver, wdir, log_fs )

        ( ok, ( how, obj, note ) ) = retuple
        if ok:
            message( [ MSG_EXIT,  how, 'install', obj, 'OK' ], log=log_fs )
            if how == 'docker':
                list_docker_images.append( obj )
            list_success.append( ( how, obj, note ) )
        else:
            list_failure.append( ( how, obj, log_path ) )
            err_msg( [ how, 'install', obj, 'NG' ], log=log_fs )

        now_str = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S' )
        message( [ SEP, now_str ], log=log_fs, out=None )
        if log_fs is not None:
            log_fs.close()

    except Exception as e:
        err_msg( [ e ] )
        message( [ '**** logfile:', log_path ] )
    except KeyboardInterrupt:
        message( [ '\n^C' ], out=sys.stderr )
        NULL_FS.close()
        sys.exit( 1 )

if list_success == [] and list_failure == []:
    # this should not happen
    NULL_FS.close()
    sys.exit( 1 )

if not FLAG_KEEP:
    message( [] )
    if list_failure == []:
        delete_work_dir( wdir )
    else:
        warn_msg( [ 'the work dir is not deleted due to the error' ] )

## Summary

yum_success    = False
docker_success = False
message( [ '\nSummary' ] )
if list_success != []:
    for ( how, obj, note ) in list_success:
        if not DRYRUN:
            message( [ 'OK\t' ], newline=False )
        else:
            message( [ '[DRY]\t' ], newline=False )
        if note == '':
            message( [ how, obj ] )
        else:
            message( [ how, obj, note ] )

        if how == 'yum':
            yum_success = True
        elif how == 'docker':
            docker_success = True

if list_skipped != []:
    for ( how, arch, centos, pip_ver ) in list_skipped:
        message( [ 'SKIPPED\t',
                   how,
                   arch,
                   'centos'+str(centos),
                   'PiP-v'+str(pip_ver) ] )

if list_noupdate != []:
    for ( how, arch, centos, pip_ver ) in list_skipped:
        message( [ 'NOUPDATE\t',
                   how,
                   arch,
                   'centos'+str(centos),
                   'PiP-v'+str(pip_ver) ] )

if list_failure != []:
    for ( how, obj, log_path ) in list_failure:
        message( [ 'NG\t' ], newline=False )
        if log_path == '':
            message( [ how, obj ] )
        else:
            message( [ how, obj, ' **LOG:', log_path ] )

if not DRYRUN:
    if yum_success or docker_success:
        message( [ '\nNote' ] )
        if yum_success:
            message( [ 'YUM: PiP is installed at /opt/process-in-process/' ] )
        if docker_success:
            message( [ 'DOCKER: PiP is installed at /opt/process-in-process.  To run PiP Docker image:' ] )
            message( [ '     $ [sudo] docker run -it <PIP_DOCKER_TAG> /bin/bash' ] )
            message( [ '     PIP_DOCKER_TAGS: ' ], newline=False )
            message( list_docker_images )

# check inter-option constraints
warn_newline = False
if ARGS.sudo:
    if not 'yum' in how_list and not 'docker' in how_list:
        if not warn_newline:
            warn_newline = True
            message( [] )
        warn_msg( [ "'--sudo' option is only effective with yum and docker installations" ] )
if not ARGS.sudo and DRYRUN:
    if 'yum' in how_list or 'docker' in how_list:
        if not warn_newline:
            warn_newline = True
            message( [] )
        warn_msg( [ "'--sudo' option is required with yum and docker installations" ] )
if ARGS.centos != '':
    if not 'docker' in how_list:
        if not warn_newline:
            warn_newline = True
            message( [] )
        warn_msg( [ "'--centos' option is only effective with docker installation" ] )
if not 'github' in how_list:
    if ARGS.noglibc:
        if not warn_newline:
            warn_newline = True
            message( [] )
        warn_msg( [ "'--noglibc' option is only effective with github installation" ] )
    if ARGS.nogdb:
        if not warn_newline:
            warn_newline = True
            message( [] )
        warn_msg( [ "'--nogdb' option is only effective with github installation" ] )
    if ARGS.notest:
        if not warn_newline:
            warn_newline = True
            message( [] )
        warn_msg( [ "'--notest' option is only effective with github installation" ] )

NULL_FS.close()
if list_failure != []:
    sys.exit( 1 )

sys.exit( 0 )