From 3206394ef82216e1244da6332948441a011e5182 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 22 Sep 2019 17:02:15 +0200 Subject: [PATCH] manylinux2014 --- .travis.yml | 58 +-- README.rst | 102 +++-- build.sh | 14 +- docker/Dockerfile-aarch64 | 19 + docker/Dockerfile-i686 | 11 +- docker/Dockerfile-x86_64 | 5 +- docker/build_scripts/build.sh | 97 ++--- docker/build_scripts/build_env.sh | 8 +- docker/build_scripts/build_utils.sh | 52 +-- docker/build_scripts/manylinux-check.py | 29 +- docker/build_scripts/py37-requirements.txt | 6 +- docker/build_scripts/requirements.txt | 6 +- docker/glibc/Dockerfile | 27 -- docker/glibc/README.rst | 79 ---- docker/glibc/build_scripts/CentOS-source.repo | 7 - docker/glibc/build_scripts/glibc.spec.patch | 29 -- .../rebuild-glibc-without-vsyscall.sh | 52 --- .../glibc/build_scripts/remove-vsyscall.patch | 401 ------------------ 18 files changed, 186 insertions(+), 816 deletions(-) create mode 100644 docker/Dockerfile-aarch64 delete mode 100644 docker/glibc/Dockerfile delete mode 100644 docker/glibc/README.rst delete mode 100644 docker/glibc/build_scripts/CentOS-source.repo delete mode 100644 docker/glibc/build_scripts/glibc.spec.patch delete mode 100644 docker/glibc/build_scripts/rebuild-glibc-without-vsyscall.sh delete mode 100644 docker/glibc/build_scripts/remove-vsyscall.patch diff --git a/.travis.yml b/.travis.yml index 25e3656b..43db321f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: c +os: linux +dist: bionic sudo: required -dist: trusty services: - docker @@ -10,48 +11,27 @@ branches: except: - /^pyup-/ -cache: - directories: - - $HOME/docker - env: global: # QUAY_USERNAME and QUAY_PASSWORD for docker image upload - secure: "lKaTzEL6UNiEfp+BWLOUILG9BMtjwEMpwt6Yag0cQGHix7qJ/ElZ0t3oFw6ZwuDmA5qceAXIdxHLUK9HGVI2MloLk8czGhjvtfJ4XhOxtEJRQ0VkDGPsKN4cfhB4ZjGo6GAPtNqStMyNiY7BZuTrZa7coDLCoUeYcOmTpi6pmd1rrkk725B9QCTuhFHbPhkuL2yu/Jk6WxkHJBKjmuZek+iQa7lRItgMrG0/319PXLvwIGGl00nLFy+Ly5Ciwzux4wuHLTySZQKu0H9FX81A7smM0FW/42kg3ckGa2qLxRw/Pi8Nm/aIk8LD0QXzI5N7HhFfidOTgDS8Mt1HgfxmTk4wUXZ/KvCCshqjimzMc/s9i9wPZX9UqqcfrpZkmwz8dzhm1bndN45ZOCy6xAYT6dzf8T4mLMDjVWSW4+DUoW4sYHRLVujjcMk7ybcwGV43VruPTJnc8XVAhT+VIMQkoPjhQmTOn8h82LRNGYtLa5RReCh9OPKVYB2Quz18FXMWgFt7A6VWudL0c7/8CusLvuo+pLcxt9pnV40rvu1YEohpEj8qR/qTSaDUBZM0J9SVf5zrZR80pZUnXkDF8nm+mcLOTley3YWipU19lCR7dzVyCAiQdVAuNPdnyem3Yk8enGkAJbfLd6eaIDs+p73D0JXh1Nx1px1movVLQH3ohIw=" - secure: "w1614pomHLltkBhqWM2bOvbymFWIWKqSqqIBDvaNn9tbQScioItJoELBT7g7+cD7nyU7OvpQ1U2fk0xVkCeNvYU0xS1vP4o/VnZRpup7f7Tkiq+2rf4fjwYr3HHnJjwak1l9bsw6FkgzKaVvSdiUJHMVxiIuLd3fVozR7qjBBhTDxSlWGOpSgd+ttpgMZwU5zQjdaVQr1D7E8M0979ZnWMrNRyLiAUeHaPILS815b+ijgqR+i5nmu0/FTCGM9Ik4KIzIfWq8AdfPdbRiq8c+LrrTPfyKcIQJaHmfduYRM4LycGWwzkXFBNtLrJ7uFLG9RDVemOHuHOWIJX8qCUIV4XuESXxH3fUQr6r+yxquTJbzXxNtoaLa6tBOTQWKDrRjT4z9Mf9Im14F2V59EUDoQowHx5bjunOH5wg3ruYNKYYBFRYra5kx0CkKrqFBzyl8fTUEQLyx1HWTVUC1WTXEeD/aFKOSIxW5DxZr5W4LLlW2+Raa52ZzY28Q6AdueFQCRzoJ70/GsJRlSsBdWNOHN4gSp1cZuToLWY15y64QhAMVDpikB+V4hmkbceLiTqeWzTStNL1sa32RHr6i/9zeFZw1pMD1+eOg9x6fgODfh2sqr/zPbu2oONsHnc4D2jwsEax4o+Dv5QHLvK7jdyWUmu47a9QReoexXK60jZXs3CA=" -# use YAML aliases and anchors to avoid duplication in stages -# c.f. https://github.com/travis-ci/travis-ci/issues/8295#issuecomment-454457787 -manylinux-build: &manylinux-build - stage: "Build manylinux images" - before_install: - # Load cached docker images - - if [[ -d $HOME/docker ]]; then ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load"; fi - script: - - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh - deploy: - provider: script - script: docker/deploy.sh - on: - branch: master - repo: pypa/manylinux - -jobs: +matrix: include: - - stage: "Patch glibc" - env: - - PLATFORM="x86_64" - before_install: - # Load cached docker images - - if [[ -d $HOME/docker ]]; then ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load"; fi - script: - - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh glibc_only - before_cache: - # Save tagged docker images - - mkdir -p $HOME/docker && docker images -a --filter='dangling=false' --format '{{.Repository}}:{{.Tag}} {{.ID}}' | grep 'centos-with-vsyscall:latest' | xargs -n 2 -t sh -c 'test -e $HOME/docker/$1.tar.gz || docker save $0 | gzip -2 > $HOME/docker/$1.tar.gz' - - <<: *manylinux-build - env: - - PLATFORM="x86_64" - - <<: *manylinux-build - env: - - PLATFORM="i686" + - arch: amd64 + env: PLATFORM="x86_64" + - arch: amd64 + env: PLATFORM="i686" + - arch: arm64 + env: PLATFORM="aarch64" + +script: + - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh + +deploy: + provider: script + script: docker/deploy.sh + on: + branch: manylinux2014 + repo: pypa/manylinux diff --git a/README.rst b/README.rst index df8fb1a1..5bb469a9 100644 --- a/README.rst +++ b/README.rst @@ -11,21 +11,48 @@ The goal of the manylinux project is to provide a convenient way to distribute binary Python extensions as wheels on Linux. This effort has produced `PEP 513 `_ which is further enhanced by `PEP 571 `_ -defining ``manylinux2010_x86_64`` and ``manylinux2010_i686`` platform tags. +and now `PEP 599 `_ defining +``manylinux2014_*`` platform tags. PEP 513 defined ``manylinux1_x86_64`` and ``manylinux1_i686`` platform tags -and the wheels were built on Centos5. Centos5 reached End of Life (EOL) on +and the wheels were built on Centos5. Centos5 reached End of Life (EOL) on March 31st, 2017 and thus PEP 571 was proposed. -Code and details regarding ``manylinux1`` can be found here: +PEP 571 defined ``manylinux2010_x86_64`` and ``manylinux2010_i686`` platform +tags and the wheels were built on Centos6. Centos6 will reach End of Life (EOL) +on November 30th, 2020. + +PEP 599 defines the following platform tags: + +- ``manylinux2014_x86_64`` + +- ``manylinux2014_i686`` + +- ``manylinux2014_aarch64`` + +- ``manylinux2014_armv7l`` + +- ``manylinux2014_ppc64`` + +- ``manylinux2014_ppc64le`` + +- ``manylinux2014_s390x`` + +Wheels are built on CentOS 7 which will reach End of Life (EOL) +on June 30th, 2024. + +Code and details regarding ``manylinux1`` can be found here: `manylinux1 `_. +Code and details regarding ``manylinux2010`` can be found here: +`manylinux2010 `_. + Wheel packages compliant with those tags can be uploaded to `PyPI `_ (for instance with `twine `_) and can be installed with -**pip 19.0 and later**. +**pip 19.3 and later**. -The manylinux2010 tags allow projects to distribute wheels that are +The manylinux2014 tags allow projects to distribute wheels that are automatically installed (and work!) on the vast majority of desktop and server Linux distributions. @@ -35,79 +62,77 @@ This repository hosts several manylinux-related things: Docker images ------------- -.. image:: https://travis-ci.org/pypa/manylinux.svg?branch=master +.. image:: https://travis-ci.org/pypa/manylinux.svg?branch=manylinux2014 :target: https://travis-ci.org/pypa/manylinux Building manylinux-compatible wheels is not trivial; as a general rule, binaries built on one Linux distro will only work on other Linux distros that are the same age or newer. Therefore, if we want to make -binaries that run on most Linux distros, we have to use a very old -distro -- CentOS 6. +binaries that run on most Linux distros, we have to use an old enough +distro -- CentOS 7. -Rather than forcing you to install CentOS 6 yourself, install Python, -etc., we provide a `Docker `_ image where we've +Rather than forcing you to install CentOS 7 yourself, install Python, +etc., we provide `Docker `_ images where we've done the work for you: -64-bit image (x86-64): ``quay.io/pypa/manylinux2010_x86_64`` +x86_64 image: ``quay.io/pypa/manylinux2014_x86_64`` + +.. image:: https://quay.io/repository/pypa/manylinux2014_x86_64/status + :target: https://quay.io/repository/pypa/manylinux2014_x86_64 -.. image:: https://quay.io/repository/pypa/manylinux2010_x86_64/status - :target: https://quay.io/repository/pypa/manylinux2010_x86_64 +i686 image: ``quay.io/pypa/manylinux2014_i686`` + +.. image:: https://quay.io/repository/pypa/manylinux2014_i686/status + :target: https://quay.io/repository/pypa/manylinux2014_i686 + +aarch64 image: ``quay.io/pypa/manylinux2014_aarch64`` + +.. image:: https://quay.io/repository/pypa/manylinux2014_aarch64/status + :target: https://quay.io/repository/pypa/manylinux2014_aarch64 These images are rebuilt using Travis-CI on every commit to this repository; see the -`docker/ `_ +`docker/ `_ directory for source code. The images currently contain: -- CPython 2.7, 3.4, 3.5, 3.6 and 3.7, installed in +- CPython 3.5, 3.6, 3.7 and 3.8, installed in ``/opt/python/-``. The directories are named after the PEP 425 tags for each environment -- - e.g. ``/opt/python/cp27-cp27mu`` contains a wide-unicode CPython 2.7 - build, and can be used to produce wheels named like - ``--cp27-cp27mu-.whl``. + e.g. ``/opt/python/cp35-cp35m`` contains a CPython 3.5 build, and + can be used to produce wheels named like + ``--cp35-cp35m-.whl``. -- Devel packages for all the libraries that PEP 571 allows you to +- Devel packages for all the libraries that PEP 599 allows you to assume are present on the host system - The `auditwheel `_ tool -Note that prior to CPython 3.3, there were two ABI-incompatible ways -of building CPython: ``--enable-unicode=ucs2`` and -``--enable-unicode=ucs4``. We provide both versions -(e.g. ``/opt/python/cp27-cp27m`` for narrow-unicode, -``/opt/python/cp27-cp27mu`` for wide-unicode). NB: essentially all -Linux distributions configure CPython in ``mu`` -(``--enable-unicode=ucs4``) mode, but ``--enable-unicode=ucs2`` builds -are also encountered in the wild. Other less common or virtually -unheard of flag combinations (such as ``--with-pydebug`` (``d``) and -``--without-pymalloc`` (absence of ``m``)) are not provided. Building Docker images ---------------------- -To build the Docker images, you will need to fetch the tarballs to -``docker/sources/`` prior to building. This can be done with the -provided prefetch script, after which you can proceed with building. -Please run the following command from the current (root) directory:: +To build the Docker images, please run the following command from the +current (root) directory: $ PLATFORM=$(uname -m) TRAVIS_COMMIT=latest ./build.sh Example ------- -An example project which builds 64-bit wheels for each Python interpreter +An example project which builds x86_64 wheels for each Python interpreter version can be found here: https://github.com/pypa/python-manylinux-demo. The -repository also contains demo to build 32-bit and 64-bit wheels with ``manylinux1`` +repository also contains demo to build i686 and x86_64 wheels with ``manylinux1`` tags. This demonstrates how to use these docker images in conjunction with auditwheel to build manylinux-compatible wheels using the free `travis ci `_ continuous integration service. -(NB: for the 32-bit images running on a 64-bit host machine, it's necessary to run +(NB: for the i686 images running on a x86_64 host machine, it's necessary to run everything under the command line program `linux32`, which changes reported architecture in new program environment. See `this example invocation `_) @@ -124,9 +149,12 @@ where the PEP was originally written, so if for some reason you really want to see the full history of edits it went through, then this is the place to look. -The proposal to upgrade ``manylinux1`` to ``manylinux2010`` after Centos5 +The proposal to upgrade ``manylinux1`` to ``manylinux2010`` after Centos5 reached EOL was discussed in `PEP 571 `_. +The proposal to upgrade ``manylinux2010`` to ``manylinux2014`` was +discussed in `PEP 599 `_. + This repo also has some analysis code that was used when putting together the original proposal in the ``policy-info/`` directory. diff --git a/build.sh b/build.sh index 680c1fc2..1f39a8a7 100755 --- a/build.sh +++ b/build.sh @@ -4,16 +4,4 @@ set -ex -if [ $PLATFORM == x86_64 ] || [ "$1" == "glibc_only" ]; then - # Output something every 10 minutes or Travis kills the job - while sleep 9m; do echo -n -e " \b"; done & - docker build --rm -t centos-with-vsyscall:latest --cache-from centos-with-vsyscall:latest --target centos-with-vsyscall -f docker/glibc/Dockerfile docker/glibc/ - # Killing background sleep loop - kill %1 - if [ "$1" == "glibc_only" ]; then - exit 0 - fi - docker build --rm -t quay.io/pypa/manylinux2010_centos-6-no-vsyscall --cache-from quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest --cache-from centos-with-vsyscall:latest -f docker/glibc/Dockerfile docker/glibc/ -fi - -docker build --rm -t quay.io/pypa/manylinux2010_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/ +docker build --rm -t quay.io/pypa/manylinux2014_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/ diff --git a/docker/Dockerfile-aarch64 b/docker/Dockerfile-aarch64 new file mode 100644 index 00000000..cbe75aa3 --- /dev/null +++ b/docker/Dockerfile-aarch64 @@ -0,0 +1,19 @@ +FROM arm64v8/centos:7 +LABEL maintainer="The ManyLinux project" + +ENV AUDITWHEEL_ARCH aarch64 +ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-8/root +ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH +ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib +ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig + +COPY build_scripts /build_scripts +RUN bash build_scripts/build.sh && rm -r build_scripts + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile-i686 b/docker/Dockerfile-i686 index 965a6662..3ce938a5 100644 --- a/docker/Dockerfile-i686 +++ b/docker/Dockerfile-i686 @@ -1,20 +1,18 @@ -FROM i386/centos:6 +FROM i386/centos:7 LABEL maintainer="The ManyLinux project" ENV AUDITWHEEL_ARCH i686 -ENV AUDITWHEEL_PLAT manylinux2010_$AUDITWHEEL_ARCH +ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-7/root -ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH -ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib64:/usr/local/lib ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig # Set a base architecture of yum package to i386 RUN echo "i386" > /etc/yum/vars/basearch -# To have linux32 command +# To have linux32 command RUN yum -y update && \ yum install -y util-linux-ng @@ -23,5 +21,4 @@ RUN linux32 bash build_scripts/build.sh && rm -r build_scripts ENV SSL_CERT_FILE=/opt/_internal/certs.pem -ENTRYPOINT ["linux32"] CMD ["/bin/bash"] diff --git a/docker/Dockerfile-x86_64 b/docker/Dockerfile-x86_64 index f8bd301d..f897adab 100644 --- a/docker/Dockerfile-x86_64 +++ b/docker/Dockerfile-x86_64 @@ -1,9 +1,8 @@ -# See docker/glibc/ -FROM quay.io/pypa/manylinux2010_centos-6-no-vsyscall +FROM centos:7 LABEL maintainer="The ManyLinux project" ENV AUDITWHEEL_ARCH x86_64 -ENV AUDITWHEEL_PLAT manylinux2010_$AUDITWHEEL_ARCH +ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 diff --git a/docker/build_scripts/build.sh b/docker/build_scripts/build.sh index cc9d7060..bf5b81f1 100755 --- a/docker/build_scripts/build.sh +++ b/docker/build_scripts/build.sh @@ -10,14 +10,14 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # Dependencies for compiling Python that we want to remove from # the final image after compiling Python -PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel" +PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel libdb-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel" -# Libraries that are allowed as part of the manylinux2010 profile -# Extract from PEP: https://www.python.org/dev/peps/pep-0571/#the-manylinux2010-policy +# Libraries that are allowed as part of the manylinux2014 profile +# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy # On RPM-based systems, they are provided by these packages: # Package: Libraries # glib2: libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0 -# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libcrypt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 +# glibc: libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6 # libICE: libICE.so.6 # libX11: libX11.so.6 # libXext: libXext.so.6 @@ -28,18 +28,17 @@ PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-d # # PEP is missing the package for libSM.so.6 for RPM based system # Install development packages (except for libgcc which is provided by gcc install) -MANYLINUX2010_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" +MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" # Get build utilities source $MY_DIR/build_utils.sh -# Prerequisite for architecture -case $AUDITWHEEL_ARCH in -x86_64) - # See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages - echo "multilib_policy=best" >> /etc/yum.conf - ;; -esac +# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages +echo "multilib_policy=best" >> /etc/yum.conf +# Error out if requested packages do not exist +echo "skip_missing_names_on_install=False" >> /etc/yum.conf +# Make sure that locale will not be removed +sed -i '/^override_install_langs=/d' /etc/yum.conf # https://hub.docker.com/_/centos/ # "Additionally, images with minor version tags that correspond to install @@ -51,62 +50,48 @@ esac # Decided not to clean at this point: https://github.com/pypa/manylinux/pull/129 yum -y update -# EPEL support (for cmake28 & yasm) -yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm - -case $AUDITWHEEL_ARCH in -x86_64) - # Install devtoolset-8 - yum -y install centos-release-scl - yum -y install \ - devtoolset-8-binutils \ - devtoolset-8-gcc \ - devtoolset-8-gcc-c++ \ - devtoolset-8-gcc-gfortran \ - ;; -i686) - # Install devtoolset-7 (binutils, gcc, gcc-c++, gcc-gfortran) - devtoolset7s=( - "devtoolset-7-runtime-7.1-4.el6.i686.rpm" - "devtoolset-7-binutils-2.28-11.el6.i686.rpm" - "devtoolset-7-gcc-7.3.1-5.10.el6.i686.rpm" - "devtoolset-7-libstdc++-devel-7.3.1-5.10.el6.i686.rpm" - "devtoolset-7-gcc-c++-7.3.1-5.10.el6.i686.rpm" - "devtoolset-7-libquadmath-devel-7.3.1-5.10.el6.i686.rpm" - "devtoolset-7-gcc-gfortran-7.3.1-5.10.el6.i686.rpm" - ) - for rpm in "${devtoolset7s[@]}"; do - yum install -y "https://www.repo.cloudlinux.com/cloudlinux/6.10/sclo/devtoolset-7/i386/$rpm" - done - ;; -esac +# upgrading glibc-common can end with removal on en_US.UTF-8 locale +localedef -i en_US -f UTF-8 en_US.UTF-8 + +DEVTOOLSET8_TOOLCHAIN_DEPS="devtoolset-8-binutils devtoolset-8-gcc devtoolset-8-gcc-c++ devtoolset-8-gcc-gfortran" +DEFAULT_TOOLCHAIN_DEPS="gcc gcc-c++ gcc-gfortran" +if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then + # Software collection (for devtoolset-8) + yum -y install centos-release-scl-rh + # EPEL support (for yasm) + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + YASM=yasm + TOOLCHAIN_DEPS=${DEVTOOLSET8_TOOLCHAIN_DEPS} +elif [ "${AUDITWHEEL_ARCH}" == "aarch64" ]; then + # Software collection (for devtoolset-8) + yum -y install centos-release-scl-rh + TOOLCHAIN_DEPS=${DEVTOOLSET8_TOOLCHAIN_DEPS} +elif [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + # No yasm, no devtoolset-8 on i686 + TOOLCHAIN_DEPS=${DEFAULT_TOOLCHAIN_DEPS} +fi # Development tools and libraries yum -y install \ automake \ bison \ bzip2 \ - cmake28 \ + ${TOOLCHAIN_DEPS} \ diffutils \ gettext \ file \ - kernel-devel-`uname -r` \ + kernel-devel \ libffi-devel \ make \ patch \ unzip \ which \ - yasm \ + ${YASM} \ ${PYTHON_COMPILE_DEPS} -# Install a git we link against system OpenSSL/Curl -yum -y install openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel +# Install git build_git $GIT_ROOT $GIT_HASH git version -yum -y erase openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel - -# Build an OpenSSL for Pythons. We'll delete this at the end. -build_openssl $OPENSSL_ROOT $OPENSSL_HASH # Install newest autoconf build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH @@ -155,8 +140,6 @@ ln -s $($PY37_BIN/python -c 'import certifi; print(certifi.where())') \ # Dockerfiles: export SSL_CERT_FILE=/opt/_internal/certs.pem -# Now we can delete our built OpenSSL headers/static libs since we've linked everything we need -rm -rf /usr/local/ssl # Install patchelf (latest with unreleased bug fixes) curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz @@ -179,7 +162,7 @@ yum -y erase \ libX11 \ wireless-tools \ ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1 -yum -y install ${MANYLINUX2010_DEPS} +yum -y install ${MANYLINUX_DEPS} yum -y clean all > /dev/null 2>&1 yum list installed @@ -211,9 +194,11 @@ find /opt/_internal -depth \ # Fix libc headers to remain compatible with C99 compilers. find /usr/include/ -type f -exec sed -i 's/\bextern _*inline_*\b/extern __inline __attribute__ ((__gnu_inline__))/g' {} + -# remove useless things that have been installed by devtoolset -rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man -find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then + # remove useless things that have been installed by devtoolset + rm -rf $DEVTOOLSET_ROOTPATH/usr/share/man + find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf +fi rm -rf /usr/share/backgrounds # if we updated glibc, we need to strip locales again... localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive diff --git a/docker/build_scripts/build_env.sh b/docker/build_scripts/build_env.sh index 423179cd..df2b5727 100644 --- a/docker/build_scripts/build_env.sh +++ b/docker/build_scripts/build_env.sh @@ -2,13 +2,7 @@ PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python # of the form .. or ..rc -CPYTHON_VERSIONS="2.7.17 3.4.10 3.5.9 3.6.9 3.7.5 3.8.0" - -# openssl version to build, with expected sha256 hash of .tar.gz -# archive. -OPENSSL_ROOT=openssl-1.0.2t -OPENSSL_HASH=14cb464efe7ac6b54799b34456bd69558a749a4931ecfd9cf9f71d7881cac7bc -OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source +CPYTHON_VERSIONS="3.5.9 3.6.9 3.7.5 3.8.0" PATCHELF_VERSION=0.10 PATCHELF_HASH=b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13 diff --git a/docker/build_scripts/build_utils.sh b/docker/build_scripts/build_utils.sh index 0997396c..e20bf654 100755 --- a/docker/build_scripts/build_utils.sh +++ b/docker/build_scripts/build_utils.sh @@ -31,27 +31,18 @@ function pyver_dist_dir { function do_cpython_build { local py_ver=$1 check_var $py_ver - local ucs_setting=$2 - check_var $ucs_setting tar -xzf Python-$py_ver.tgz pushd Python-$py_ver - if [ "$ucs_setting" = "none" ]; then - unicode_flags="" - dir_suffix="" - else - local unicode_flags="--enable-unicode=$ucs_setting" - local dir_suffix="-$ucs_setting" - fi - local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}" + local prefix="/opt/_internal/cpython-${py_ver}" mkdir -p ${prefix}/lib - ./configure --prefix=${prefix} --disable-shared $unicode_flags > /dev/null - make -j2 > /dev/null - make install > /dev/null + ./configure --prefix=${prefix} --disable-shared > /dev/null + make -j$(nproc) > /dev/null + make -j$(nproc) install > /dev/null popd rm -rf Python-$py_ver # Some python's install as bin/python3. Make them available as # bin/python. - if [ -e ${prefix}/bin/python3 ]; then + if [ -e ${prefix}/bin/python3 ] && [ ! -e ${prefix}/bin/python ]; then ln -s python3 ${prefix}/bin/python fi ${prefix}/bin/python get-pip.py @@ -74,12 +65,7 @@ function build_cpython { curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz.asc gpg --verify Python-$py_ver.tgz.asc - if [ $(lex_pyver $py_ver) -lt $(lex_pyver 3.3) ]; then - do_cpython_build $py_ver ucs2 - do_cpython_build $py_ver ucs4 - else - do_cpython_build $py_ver none - fi + do_cpython_build $py_ver rm -f Python-$py_ver.tgz rm -f Python-$py_ver.tgz.asc } @@ -102,13 +88,6 @@ function build_cpythons { } -function do_openssl_build { - ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null - make > /dev/null - make install_sw > /dev/null -} - - function fetch_source { # This is called both inside and outside the build context (e.g. in Travis) to prefetch # source tarballs, where curl exists (and works) @@ -136,19 +115,6 @@ function check_sha256sum { } -function build_openssl { - local openssl_fname=$1 - check_var ${openssl_fname} - local openssl_sha256=$2 - check_var ${openssl_sha256} - fetch_source ${openssl_fname}.tar.gz ${OPENSSL_DOWNLOAD_URL} - check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256} - tar -xzf ${openssl_fname}.tar.gz - (cd ${openssl_fname} && do_openssl_build) - rm -rf ${openssl_fname} ${openssl_fname}.tar.gz -} - - function build_git { local git_fname=$1 check_var ${git_fname} @@ -158,15 +124,15 @@ function build_git { fetch_source v${git_fname}.tar.gz ${GIT_DOWNLOAD_URL} check_sha256sum v${git_fname}.tar.gz ${git_sha256} tar -xzf v${git_fname}.tar.gz - (cd git-${git_fname} && make install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null) + (cd git-${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null) rm -rf git-${git_fname} v${git_fname}.tar.gz } function do_standard_install { ./configure "$@" > /dev/null - make > /dev/null - make install > /dev/null + make -j$(nproc) > /dev/null + make -j$(nproc) install > /dev/null } diff --git a/docker/build_scripts/manylinux-check.py b/docker/build_scripts/manylinux-check.py index d030a9ad..a215f11e 100644 --- a/docker/build_scripts/manylinux-check.py +++ b/docker/build_scripts/manylinux-check.py @@ -1,24 +1,33 @@ -# Logic copied from PEP 513 +# Logic copied from PEP 599 import sys -def is_manylinux2010_compatible(): - # Only Linux, and only x86-64 / i686 +def is_manylinux2014_compatible(): + # Only Linux, and only supported architectures from distutils.util import get_platform - if get_platform() not in ["linux-x86_64", "linux-i686"]: + if get_platform() not in [ + "linux-x86_64", + "linux-i686", + "linux-aarch64", + "linux-armv7l", + "linux-ppc64", + "linux-ppc64le", + "linux-s390x", + ]: return False # Check for presence of _manylinux module try: import _manylinux - return bool(_manylinux.manylinux1_compatible) + return bool(_manylinux.manylinux2014_compatible) except (ImportError, AttributeError): # Fall through to heuristic check below pass - # Check glibc version. CentOS 6 uses glibc 2.12. - return have_compatible_glibc(2, 12) + # Check glibc version. CentOS 7 uses glibc 2.17. + # PEP 513 contains an implementation of this function. + return have_compatible_glibc(2, 17) def have_compatible_glibc(major, minimum_minor): @@ -49,9 +58,9 @@ def have_compatible_glibc(major, minimum_minor): return True -if is_manylinux2010_compatible(): - print("%s is manylinux2010 compatible" % (sys.executable,)) +if is_manylinux2014_compatible(): + print("%s is manylinux2014 compatible" % (sys.executable,)) sys.exit(0) else: - print("%s is NOT manylinux2010 compatible" % (sys.executable,)) + print("%s is NOT manylinux2014 compatible" % (sys.executable,)) sys.exit(1) diff --git a/docker/build_scripts/py37-requirements.txt b/docker/build_scripts/py37-requirements.txt index 21199cdd..60fa46e5 100644 --- a/docker/build_scripts/py37-requirements.txt +++ b/docker/build_scripts/py37-requirements.txt @@ -3,9 +3,9 @@ certifi==2019.9.11 \ --hash=sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50 \ --hash=sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef -auditwheel==2.1.1 \ - --hash=sha256:d9f0f47c0ecdf4f72316eabfb19223b81aec95f0deba97729e9c09ea4f2c166e \ - --hash=sha256:f60e265dc168ae7c9d65a814410881c5ca170d88644c178f8e3aa20eecd0fae1 +auditwheel==3.0.0.0rc1 \ + --hash=sha256:23efd534d798e0f7a8a412817d4edb471e8661b0928b30b7beae4a27bc26f5b7 \ + --hash=sha256:51d609efdeffb5d5c127e35dffc1c4116251ae7de6802e8d2697e4c2c0d37c7a # this package required for auditwheel pyelftools==0.25 \ --hash=sha256:89c6da6f56280c37a5ff33468591ba9a124e17d71fe42de971818cbff46c1b24 diff --git a/docker/build_scripts/requirements.txt b/docker/build_scripts/requirements.txt index ad47555a..a135290b 100644 --- a/docker/build_scripts/requirements.txt +++ b/docker/build_scripts/requirements.txt @@ -1,8 +1,8 @@ # pip requirements for all cpythons # NOTE: pip has GPG signatures; could download and verify independently. -pip==19.1.1 \ - --hash=sha256:44d3d7d3d30a1eb65c7e5ff1173cdf8f7467850605ac7cc3707b6064bddd0958 \ - --hash=sha256:993134f0475471b91452ca029d4390dc8f298ac63a712814f101cd1b6db46676 +pip==19.3.1 \ + --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \ + --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7 wheel==0.31.1 \ --hash=sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f \ --hash=sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c diff --git a/docker/glibc/Dockerfile b/docker/glibc/Dockerfile deleted file mode 100644 index b345fabb..00000000 --- a/docker/glibc/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM centos:6 as centos-with-vsyscall - -COPY ./build_scripts /build_scripts -RUN bash /build_scripts/rebuild-glibc-without-vsyscall.sh - -FROM centos:6 -LABEL maintainer="The Manylinux project" - -# do not install debuginfo -COPY --from=centos-with-vsyscall \ - /rpms/glibc-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/glibc-common-2.12-1.212.1.el6.x86_64.rpm \ - #/rpms/glibc-debuginfo-2.12-1.212.1.el6.x86_64.rpm \ - #/rpms/glibc-debuginfo-common-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/glibc-devel-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/glibc-headers-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/glibc-static-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/glibc-utils-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/nscd-2.12-1.212.1.el6.x86_64.rpm \ - /rpms/ - -RUN yum -y install /rpms/* && rm -rf /rpms && yum -y clean all && rm -rf /var/cache/yum/* && \ - # if we updated glibc, we need to strip locales again... - localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive && \ - mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl && \ - build-locale-archive && \ - find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf diff --git a/docker/glibc/README.rst b/docker/glibc/README.rst deleted file mode 100644 index c4b0fc37..00000000 --- a/docker/glibc/README.rst +++ /dev/null @@ -1,79 +0,0 @@ -centos-6-no-vsyscall -==================== - -*Summary*: Because of -https://mail.python.org/pipermail/wheel-builders/2016-December/000239.html, -this a CentOS 6.10 Docker image that rebuilds ``glibc`` without -*vsyscall* is necessary to reliably run ``manylinux2010`` on 64-bit -hosts. This requires building the image on a system with -``vsyscall=emulate`` but allows the resulting container to run on -systems with ``vsyscall=none`` or ``vsyscall=emulate``. - -*vsyscall* is an antiquated optimization for a small number of -frequently-used system calls. A vsyscall-enabled Linux kernel maps a -read-only page of data and system calls into a process' memory at a -fixed address. These system calls can then be invoked by -dereferencing a function pointers to fixed offsets in that page, -saving a relatively expensive context switch. [1]_ - -Unfortunately, because the code and its location in memory are fixed -and well-known, the vsyscall mechanism has become a source of gadgets -for ROP attacks (specifically, Sigreturn-Oriented Programs). [2]_ -Linux 3.1 introduced vsyscall emulation that prevents attackers from -jumping into the middle of the system calls' code at the expense of -speed, as well as the ability to disable it entirely. [3]_ [4]_ The -vsyscall mechanism could not be eliminated at the time because -``glibc`` versions earlier than 2.14 contained hard-coded references -to the fixed memory address, specifically in ``time(2)``. [5]_ These -segfault when attempting to issue a vsyscall-optimized system call -against a kernel that has disabled it. - -Linux introduced a "virtual dynamic shared object" (vDSO) that -achieves the same high-speed, in-process system call mechanism via -shared objects sometime before the kernel's migration to git. While -old itself, vDSO 's presentation as a shared library allows it to -benefit from ASLR on modern systems, making it no more amenable to ROP -gadgets than any other shared library. ``glibc`` only switched over -completely to vDSO as of glibc 2.25, so until recently vsyscall -emulation has remained on for most kernels. [6]_ Furthermore, i686 -does not use vsyscall at all, so no version of ``glibc`` requires -patching on that architecture. - -At the same time, vsyscall emulation still exposed values useful to -ROP attacks, so Linux 4.4 added a compilation option to disable -it. [7]_ [8]_ Distributions are beginning to ship kernels configured -without vsyscall, and running CentOS 5 (``glibc`` 2.5) or 6 (``glibc`` -2.12) Docker containers on these distributions indeed causes segfaults -without ``vsyscall=emulate`` [9]_ [10]_. CentOS 6, however, is -supported until 2020. It is likely that more and more distributions -will ship with ``CONFIG_LEGACY_VSYSCALL_NONE``; if managed CI services -like Travis make this switch, developers will be unable to build -``manylinux2010`` wheels with our Docker image. - -Fortunately, vsyscall is merely an optimization, and patches that -remove it can be backported to glibc 2.12 and the library recompiled. -The result is this Docker image. It can be run on kernels regardless -of their vsyscall configuration because executable and libraries on -CentOS are dynamically linked against glibc. Libraries built on this -image are unaffected because: - -a) the kernel only maps vsyscall pages into processes; -b) only glibc used the vsyscall interface directly, and it's - included in manylinux2010's whitelist policy. - -Developers who build this vsyscall-less Docker image itself, however, -must do so on a system with ``vsyscall=emulate``. - -References: -=========== - -.. [1] https://lwn.net/Articles/446528/ -.. [2] http://www.cs.vu.nl/~herbertb/papers/srop_sp14.pdf -.. [3] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=5cec93c216db77c45f7ce970d46283bcb1933884 -.. [4] https://www.kernel.org/pub/linux/kernel/v3.x/ChangeLog-3.1 -.. [5] https://sourceware.org/git/?p=glibc.git;a=blob;f=ChangeLog;h=3a6abda7d07fdaa367c48a9274cc1c08498964dc;hb=356f8bc660a154a07b03da7c536831da5c8f74fe -.. [6] https://sourceware.org/git/?p=glibc.git;a=blob;f=ChangeLog;h=6037fef737f0338a84c6fb564b3b8dc1b1221087;hb=58557c229319a3b8d2eefdb62e7df95089eabe37 -.. [7] https://googleprojectzero.blogspot.fr/2015/08/three-bypasses-and-fix-for-one-of.html -.. [8] https://outflux.net/blog/archives/2016/09/27/security-things-in-linux-v4-4/ -.. [9] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=852620#20 -.. [10] https://github.com/CentOS/sig-cloud-instance-images/issues/62 diff --git a/docker/glibc/build_scripts/CentOS-source.repo b/docker/glibc/build_scripts/CentOS-source.repo deleted file mode 100644 index 5501f550..00000000 --- a/docker/glibc/build_scripts/CentOS-source.repo +++ /dev/null @@ -1,7 +0,0 @@ -[base-source] -name=CentOS-6.10 - Base SRPMS -baseurl=http://vault.centos.org/6.10/os/Source/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 -priority=1 -enabled=1 diff --git a/docker/glibc/build_scripts/glibc.spec.patch b/docker/glibc/build_scripts/glibc.spec.patch deleted file mode 100644 index c7a3b0f9..00000000 --- a/docker/glibc/build_scripts/glibc.spec.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/SPECS/glibc.spec b/SPECS/glibc.spec -index 9bd07c9..c389711 100644 ---- a/SPECS/glibc.spec -+++ b/SPECS/glibc.spec -@@ -1,6 +1,6 @@ - %define glibcsrcdir glibc-2.12-2-gc4ccff1 - %define glibcversion 2.12 --%define glibcrelease 1.212%{?dist} -+%define glibcrelease 1.212.1%{?dist} - %define run_glibc_tests 1 - %define auxarches athlon sparcv9v sparc64v alphaev6 - %define xenarches i686 athlon -@@ -279,6 +279,7 @@ - Patch247: glibc-rh1452717-4.patch - Patch248: glibc-rh1504810-1.patch - Patch249: glibc-rh1504810-2.patch -+Patch250: remove-vsyscall.patch - - Buildroot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) - Obsoletes: glibc-profile < 2.4 -@@ -731,6 +732,7 @@ - %patch247 -p1 - %patch248 -p1 - %patch249 -p1 -+%patch250 -E -p3 - - # A lot of programs still misuse memcpy when they have to use - # memmove. The memcpy implementation below is not tolerant at - diff --git a/docker/glibc/build_scripts/rebuild-glibc-without-vsyscall.sh b/docker/glibc/build_scripts/rebuild-glibc-without-vsyscall.sh deleted file mode 100644 index 9b2b570c..00000000 --- a/docker/glibc/build_scripts/rebuild-glibc-without-vsyscall.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -# Prep script for x86_64 that recompiles glibc without vsyscalls. - -# Stop at any error, show all commands -set -ex - -# Locate the prep directory -MY_DIR=/$(dirname "${BASH_SOURCE[0]}") - -# glibc versions -ORIGINAL_GLIBC_VERSION=2.12-1.212 -PATCHED_GLIBC_VERSION=2.12-1.212.1 - -# Source RPM topdir -SRPM_TOPDIR=/root/rpmbuild - -# Source RPM download directory -DOWNLOADED_SRPMS=/root/srpms - -# Include the CentOS source RPM repository. -# https://bugs.centos.org/view.php?id=1646 -cp $MY_DIR/CentOS-source.repo /etc/yum.repos.d/CentOS-source.repo - -# Extract and prepare the source -# https://blog.packagecloud.io/eng/2015/04/20/working-with-source-rpms/ -yum -y update -yum -y install yum-utils rpm-build -yum-builddep -y glibc -mkdir $DOWNLOADED_SRPMS -# The glibc RPM's contents are owned by mockbuild -adduser mockbuild -# yumdownloader assumes the current working directory -(cd $DOWNLOADED_SRPMS && yumdownloader --source glibc) -rpm -ivh $DOWNLOADED_SRPMS/glibc-$ORIGINAL_GLIBC_VERSION.el6.src.rpm -# Prepare the source by applying Red Hat and CentOS patches -rpmbuild -bp $SRPM_TOPDIR/SPECS/glibc.spec - -# Copy the vsyscall removal patch into place -cp $MY_DIR/remove-vsyscall.patch $SRPM_TOPDIR/SOURCES -# Patch the RPM spec file so that it uses the vsyscall removal patch -(cd $SRPM_TOPDIR/SPECS && patch -p2 < $MY_DIR/glibc.spec.patch) - -# Build the RPMS -# In case of error, you can `docker commit` to inspect the build.log -rpmbuild -ba $SRPM_TOPDIR/SPECS/glibc.spec >> /var/log/build.log - -mv $SRPM_TOPDIR/RPMS/* /rpms/ - -# Show us what happened last before cleaning up the log -echo ~~~~~~~~~~~~~~~~~~~~~ final lines of the build log ~~~~~~~~~~~~~~~~~~~~~ >/dev/null -tail -n30 /var/log/build.log -rm /var/log/build.log diff --git a/docker/glibc/build_scripts/remove-vsyscall.patch b/docker/glibc/build_scripts/remove-vsyscall.patch deleted file mode 100644 index 15f4fdcc..00000000 --- a/docker/glibc/build_scripts/remove-vsyscall.patch +++ /dev/null @@ -1,401 +0,0 @@ -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S -index 22beaec..d1e29da 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S -+++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S -@@ -68,10 +68,6 @@ - #endif - - --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- -- - .globl __lll_lock_wait_private - .type __lll_lock_wait_private,@function - .hidden __lll_lock_wait_private -@@ -250,10 +246,9 @@ __lll_timedlock_wait: - /* Get current time. */ - movq %rsp, %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- /* This is a regular function call, all caller-save registers -- might be clobbered. */ -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax -@@ -402,8 +397,9 @@ __lll_timedwait_tid: - /* Get current time. */ - 2: movq %rsp, %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S -index b6537f9..cf9121d 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S -+++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S -@@ -51,9 +51,6 @@ - orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg - #endif - --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- - - .globl __lll_robust_lock_wait - .type __lll_robust_lock_wait,@function -@@ -220,10 +217,9 @@ __lll_robust_timedlock_wait: - /* Get current time. */ - movq %rsp, %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- /* This is a regular function call, all caller-save registers -- might be clobbered. */ -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S -index fecaa50..9ea8353 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S -+++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S -@@ -26,9 +26,6 @@ - - #include - --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- - - .text - -@@ -490,13 +487,11 @@ __pthread_cond_timedwait: - movq __vdso_clock_gettime@GOTPCREL(%rip), %rax - movq (%rax), %rax - PTR_DEMANGLE (%rax) -- jz 26f - call *%rax -- jmp 27f --# endif --26: movl $__NR_clock_gettime, %eax -+# else -+ movl $__NR_clock_gettime, %eax - syscall --27: -+# endif - # ifndef __ASSUME_POSIX_TIMERS - cmpq $-ENOSYS, %rax - je 19f -@@ -510,8 +505,9 @@ __pthread_cond_timedwait: - # else - leaq 24(%rsp), %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 40(%rsp), %rax -@@ -648,8 +644,9 @@ __pthread_cond_timedwait: - /* clock_gettime not available. */ - 19: leaq 32(%rsp), %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 40(%rsp), %rax -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S -index 22a4744..f65d976 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S -+++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S -@@ -23,10 +23,6 @@ - #include - #include - -- --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- - .text - - .globl pthread_rwlock_timedrdlock -@@ -123,8 +119,9 @@ pthread_rwlock_timedrdlock: - /* Get current time. */ - movq %rsp, %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S -index 20a9c00..4338e02 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S -+++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S -@@ -23,10 +23,6 @@ - #include - #include - -- --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- - .text - - .globl pthread_rwlock_timedwrlock -@@ -120,8 +116,9 @@ pthread_rwlock_timedwrlock: - /* Get current time. */ - movq %rsp, %rdi - xorl %esi, %esi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- callq *%rax -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S -index c339494..30e67e2 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S -+++ b/BUILD/glibc-2.12-2-gc4ccff1/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S -@@ -24,10 +24,6 @@ - #include - #include - -- --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- - .text - - .globl sem_timedwait -@@ -212,9 +208,10 @@ sem_timedwait: - addq $1, NWAITERS(%r12) - - 7: xorl %esi, %esi -- movq %rsp, %rdi -- movq $VSYSCALL_ADDR_vgettimeofday, %rax -- callq *%rax -+ movq %rsp,%rdi -+ /* This call works because we directly jump to a system call entry -+ which preserves all the registers. */ -+ call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/gettimeofday.S b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/gettimeofday.S -deleted file mode 100644 -index 18ec6db..0000000 ---- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/gettimeofday.S -+++ /dev/null -@@ -1,50 +0,0 @@ --/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc. -- This file is part of the GNU C Library. -- -- The GNU C Library is free software; you can redistribute it and/or -- modify it under the terms of the GNU Lesser General Public -- License as published by the Free Software Foundation; either -- version 2.1 of the License, or (at your option) any later version. -- -- The GNU C Library is distributed in the hope that it will be useful, -- but WITHOUT ANY WARRANTY; without even the implied warranty of -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- Lesser General Public License for more details. -- -- You should have received a copy of the GNU Lesser General Public -- License along with the GNU C Library; if not, write to the Free -- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -- 02111-1307 USA. */ -- --#include --#define _ERRNO_H 1 --#include -- --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -- -- --ENTRY (__gettimeofday) -- /* Align stack. */ -- sub $0x8, %rsp -- cfi_adjust_cfa_offset(8) --#ifdef SHARED -- movq __vdso_gettimeofday(%rip), %rax -- PTR_DEMANGLE (%rax) --#else -- movq $VSYSCALL_ADDR_vgettimeofday, %rax --#endif -- callq *%rax -- /* Check error return. */ -- cmpl $-4095, %eax -- jae SYSCALL_ERROR_LABEL -- --L(pseudo_end): -- add $0x8, %rsp -- cfi_adjust_cfa_offset(-8) -- ret --PSEUDO_END(__gettimeofday) -- --libc_hidden_def (__gettimeofday) --weak_alias (__gettimeofday, gettimeofday) --libc_hidden_weak (gettimeofday) -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c -index ead7dbc..08c1ef7 100644 ---- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c -+++ b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/init-first.c -@@ -17,6 +17,7 @@ - 02111-1307 USA. */ - - #ifdef SHARED -+# include - # include - # include - -@@ -26,6 +27,8 @@ long int (*__vdso_clock_gettime) (clockid_t, struct timespec *) - __attribute__ ((nocommon)); - strong_alias (__vdso_clock_gettime, __GI___vdso_clock_gettime attribute_hidden) - -+extern int __gettimeofday (struct timeval *__tv, struct timezone *__tz); -+ - - static inline void - _libc_vdso_platform_setup (void) -@@ -33,10 +36,9 @@ _libc_vdso_platform_setup (void) - PREPARE_VERSION (linux26, "LINUX_2.6", 61765110); - - void *p = _dl_vdso_vsym ("gettimeofday", &linux26); -- /* If the vDSO is not available we fall back on the old vsyscall. */ --#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000ul -+ /* If the vDSO is not available we fall back on the syscall. */ - if (p == NULL) -- p = (void *) VSYSCALL_ADDR_vgettimeofday; -+ p = __gettimeofday; - PTR_MANGLE (p); - __vdso_gettimeofday = p; - -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/sched_getcpu.S b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/sched_getcpu.S -deleted file mode 100644 -index a950990..0000000 ---- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/sched_getcpu.S -+++ /dev/null -@@ -1,50 +0,0 @@ --/* Copyright (C) 2007 Free Software Foundation, Inc. -- This file is part of the GNU C Library. -- -- The GNU C Library is free software; you can redistribute it and/or -- modify it under the terms of the GNU Lesser General Public -- License as published by the Free Software Foundation; either -- version 2.1 of the License, or (at your option) any later version. -- -- The GNU C Library is distributed in the hope that it will be useful, -- but WITHOUT ANY WARRANTY; without even the implied warranty of -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- Lesser General Public License for more details. -- -- You should have received a copy of the GNU Lesser General Public -- License along with the GNU C Library; if not, write to the Free -- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -- 02111-1307 USA. */ -- --#include --#include --#define _ERRNO_H 1 --#include -- --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vgetcpu 0xffffffffff600800 -- -- --ENTRY (sched_getcpu) -- /* Align stack and create local variable for result. */ -- sub $0x8, %rsp -- cfi_adjust_cfa_offset(8) -- -- movq %rsp, %rdi -- xorl %esi, %esi -- movl $VGETCPU_CACHE_OFFSET, %edx -- addq %fs:0, %rdx -- -- movq $VSYSCALL_ADDR_vgetcpu, %rax -- callq *%rax -- -- cmpq $-4095, %rax -- jae SYSCALL_ERROR_LABEL -- -- movl (%rsp), %eax -- --L(pseudo_end): -- add $0x8, %rsp -- cfi_adjust_cfa_offset(-8) -- ret --PSEUDO_END(sched_getcpu) -diff --git a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/time.S b/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/time.S -deleted file mode 100644 -index e3f3268..0000000 ---- a/BUILD/glibc-2.12-2-gc4ccff1/sysdeps/unix/sysv/linux/x86_64/time.S -+++ /dev/null -@@ -1,42 +0,0 @@ --/* Copyright (C) 2001,02, 2003 Free Software Foundation, Inc. -- This file is part of the GNU C Library. -- -- The GNU C Library is free software; you can redistribute it and/or -- modify it under the terms of the GNU Lesser General Public -- License as published by the Free Software Foundation; either -- version 2.1 of the License, or (at your option) any later version. -- -- The GNU C Library is distributed in the hope that it will be useful, -- but WITHOUT ANY WARRANTY; without even the implied warranty of -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- Lesser General Public License for more details. -- -- You should have received a copy of the GNU Lesser General Public -- License along with the GNU C Library; if not, write to the Free -- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA -- 02111-1307 USA. */ -- --#include --#define _ERRNO_H 1 --#include -- --/* For the calculation see asm/vsyscall.h. */ --#define VSYSCALL_ADDR_vtime 0xffffffffff600400 -- -- --/* Return the current time as a `time_t' and also put it in *T if T is -- not NULL. Time is represented as seconds from Jan 1 00:00:00 1970. */ -- --ENTRY (time) -- /* Align stack. */ -- sub $0x8, %rsp -- cfi_adjust_cfa_offset(8) -- -- movq $VSYSCALL_ADDR_vtime, %rax -- callq *%rax -- -- add $0x8, %rsp -- cfi_adjust_cfa_offset(-8) -- ret --PSEUDO_END_NOERRNO(time) --libc_hidden_def (time)