From 723dd95f599178101cb65f556a3f5933e9ac402a Mon Sep 17 00:00:00 2001 From: Jonathan Calmels Date: Sat, 25 Jan 2020 23:22:07 -0800 Subject: [PATCH] Optimize enroot import and switch to zstd for layers compression --- conf/enroot.conf.in | 3 +++ doc/cmd/import.md | 1 + enroot.in | 1 + pkg/deb/control | 1 + pkg/rpm/SPECS/enroot.spec | 2 +- src/docker.sh | 55 +++++++++++++++++++++++++++++---------- 6 files changed, 48 insertions(+), 15 deletions(-) diff --git a/conf/enroot.conf.in b/conf/enroot.conf.in index 51dc34e..baba514 100644 --- a/conf/enroot.conf.in +++ b/conf/enroot.conf.in @@ -9,6 +9,9 @@ # Gzip program used to uncompress digest layers. #ENROOT_GZIP_PROGRAM gzip +# Options passed to zstd to compress digest layers. +#ENROOT_ZSTD_OPTIONS -1 + # Options passed to mksquashfs to produce container images. #ENROOT_SQUASH_OPTIONS -comp lzo -noD diff --git a/doc/cmd/import.md b/doc/cmd/import.md index b190015..54324a2 100644 --- a/doc/cmd/import.md +++ b/doc/cmd/import.md @@ -50,6 +50,7 @@ Requires the Docker CLI to communicate with the Docker daemon. | Setting | Default | Description | | ------ | ------ | ------ | | `ENROOT_GZIP_PROGRAM` | `pigz` _or_ `gzip` | Gzip program used to uncompress digest layers | +| `ENROOT_ZSTD_OPTIONS` | `-1` | Options passed to zstd to compress digest layers | | `ENROOT_SQUASH_OPTIONS` | `-comp lzo -noD` | Options passed to mksquashfs to produce container images | | `ENROOT_MAX_PROCESSORS` | `$(nproc)` | Maximum number of processors to use for parallel tasks (0 means unlimited) | | `ENROOT_MAX_CONNECTIONS` | `10` | Maximum number of concurrent connections (0 means unlimited) | diff --git a/enroot.in b/enroot.in index 50fe938..e31d8ba 100644 --- a/enroot.in +++ b/enroot.in @@ -70,6 +70,7 @@ config::export ENROOT_DATA_PATH "${XDG_DATA_HOME}/enroot" config::export ENROOT_TEMP_PATH "${TMPDIR:-/tmp}" config::export ENROOT_GZIP_PROGRAM "$(command -v pigz > /dev/null && echo pigz || echo gzip)" +config::export ENROOT_ZSTD_OPTIONS "-1" config::export ENROOT_SQUASH_OPTIONS "-comp lzo -noD" config::export ENROOT_MAX_PROCESSORS "$(nproc)" config::export ENROOT_MAX_CONNECTIONS 10 diff --git a/pkg/deb/control b/pkg/deb/control index 3e899c1..d96c5c1 100644 --- a/pkg/deb/control +++ b/pkg/deb/control @@ -18,6 +18,7 @@ Depends: ${shlibs:Depends}, ${misc:Depends}, parallel, passwd, squashfs-tools, + zstd, # coreutils, # findutils, # grep, diff --git a/pkg/rpm/SPECS/enroot.spec b/pkg/rpm/SPECS/enroot.spec index 196ef56..9ccd470 100644 --- a/pkg/rpm/SPECS/enroot.spec +++ b/pkg/rpm/SPECS/enroot.spec @@ -18,7 +18,7 @@ Summary: Unprivileged container sandboxing utility Conflicts: enroot %endif Requires: bash >= 4.2, curl, gawk, jq >= 1.5, parallel, shadow-utils, squashfs-tools -Requires: coreutils, grep, findutils, gzip, glibc-common, sed, tar, util-linux +Requires: coreutils, grep, findutils, gzip, glibc-common, sed, tar, util-linux, zstd #Recommends: pv, pigz, ncurses #Suggests: libnvidia-container-tools, squashfuse, fuse-overlayfs %description diff --git a/src/docker.sh b/src/docker.sh index afa19c9..7e6d239 100644 --- a/src/docker.sh +++ b/src/docker.sh @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +source "${ENROOT_LIBRARY_PATH}/common.sh" + readonly token_dir="${ENROOT_CACHE_PATH}/.tokens.${EUID}" readonly creds_file="${ENROOT_CONFIG_PATH}/.credentials" @@ -80,6 +82,36 @@ docker::_authenticate() { fi } +docker::_download_extract() ( + local -r digest="$1"; shift + local curl_args=("$@") + local tmpfile= checksum= + + set -euo pipefail + shopt -s lastpipe + umask 037 + + [ -e "${ENROOT_CACHE_PATH}/${digest}" ] && exit 0 + + trap 'common::rmall "${tmpfile}" 2> /dev/null' EXIT + tmpfile=$(mktemp -p "${ENROOT_CACHE_PATH}" "${digest}.XXXXXXXXXX") + + exec {stdout}>&1 + { + curl "${curl_args[@]}" | tee "/proc/self/fd/${stdout}" \ + | "${ENROOT_GZIP_PROGRAM}" -d -f -c \ + | zstd -T"$(expr "${ENROOT_MAX_PROCESSORS}" / "${ENROOT_MAX_CONNECTIONS}" \| 1)" -q -f -o "${tmpfile}" ${ENROOT_ZSTD_OPTIONS} + } {stdout}>&1 | sha256sum | common::read -r checksum x + exec {stdout}>&- + + if [ "${digest}" != "${checksum}" ]; then + printf "Checksum mismatch: %s\n" "${digest}" >&2 + exit 1 + fi + + mv -n "${tmpfile}" "${ENROOT_CACHE_PATH}/${digest}" +) + docker::_download() { local -r user="$1" registry="${2:-registry-1.docker.io}" tag="${4:-latest}" arch="$5" local image="$3" @@ -133,20 +165,14 @@ docker::_download() { | readarray -t missing_digests fi - # Download digests, verify their checksums and put them in cache. + # Download digests, verify their checksums and extract them in the cache. if [ "${#missing_digests[@]}" -gt 0 ]; then - common::log INFO "Downloading ${#missing_digests[@]} missing digests..." NL - parallel --plain ${TTY_ON+--bar} --retries 3 -j "${ENROOT_MAX_CONNECTIONS}" -q curl "${curl_opts[@]}" -f -o {} "${req_params[@]}" -- \ - "${url_digest}sha256:{}" ::: "${missing_digests[@]}" - common::log - - common::log INFO "Validating digest checksums..." NL - parallel --plain -j "${ENROOT_MAX_PROCESSORS}" 'sha256sum -c <<< "{} {}"' ::: "${missing_digests[@]}" >&2 + common::log INFO "Downloading ${#missing_digests[@]} missing layers..." NL + BASH_ENV="${BASH_SOURCE[0]}" parallel --plain ${TTY_ON+--bar} --shuf --retries 2 -j "${ENROOT_MAX_CONNECTIONS}" -q \ + docker::_download_extract "{}" "${curl_opts[@]}" -f "${req_params[@]}" -- "${url_digest}sha256:{}" ::: "${missing_digests[@]}" common::log - chmod 640 "${missing_digests[@]}" - mv -n "${missing_digests[@]}" "${ENROOT_CACHE_PATH}" else - common::log INFO "Found all digests in cache" + common::log INFO "Found all layers in cache" fi # Return the container configuration along with all the layers. @@ -216,7 +242,7 @@ docker::import() ( local filename="$2" arch="$3" local layers=() config= image= registry= tag= user= tmpdir= - common::checkcmd curl grep awk jq parallel tar "${ENROOT_GZIP_PROGRAM}" find mksquashfs + common::checkcmd curl grep awk jq parallel tar "${ENROOT_GZIP_PROGRAM}" find mksquashfs zstd # Parse the image reference of the form 'docker://[@][#][:]'. local -r reg_user="[[:alnum:]_.!~*\'()%\;:\&=+$,-@]+" @@ -273,7 +299,7 @@ docker::import() ( # Extract all the layers locally. common::log INFO "Extracting image layers..." NL parallel --plain ${TTY_ON+--bar} -j "${ENROOT_MAX_PROCESSORS}" mkdir {\#}\; tar -C {\#} --warning=no-timestamp --anchored --exclude='dev/*' \ - --use-compress-program=\'"${ENROOT_GZIP_PROGRAM}"\' -pxf \'"${ENROOT_CACHE_PATH}/{}"\' ::: "${layers[@]}" + --use-compress-program=zstd -pxf \'"${ENROOT_CACHE_PATH}/{}"\' ::: "${layers[@]}" common::fixperms . common::log @@ -284,7 +310,8 @@ docker::import() ( # Configure the rootfs. mkdir 0 - docker::configure "${PWD}/0" "${ENROOT_CACHE_PATH}/${config}" "${arch}" + zstd -q -d -o config "${ENROOT_CACHE_PATH}/${config}" + docker::configure "${PWD}/0" config "${arch}" # Create the final squashfs filesystem by overlaying all the layers. common::log INFO "Creating squashfs filesystem..." NL