Optimize enroot import and switch to zstd for layers compression

conf/enroot.conf.in

@@ -9,6 +9,9 @@
 # Gzip program used to uncompress digest layers.
 #ENROOT_GZIP_PROGRAM        gzip
 
+# Options passed to zstd to compress digest layers.
+#ENROOT_ZSTD_OPTIONS        -1
+
 # Options passed to mksquashfs to produce container images.
 #ENROOT_SQUASH_OPTIONS      -comp lzo -noD
 

doc/cmd/import.md

@@ -50,6 +50,7 @@ Requires the Docker CLI to communicate with the Docker daemon.
 | Setting | Default | Description |
 | ------ | ------ | ------ |
 | `ENROOT_GZIP_PROGRAM` | `pigz` _or_ `gzip` | Gzip program used to uncompress digest layers |
+| `ENROOT_ZSTD_OPTIONS` | `-1` | Options passed to zstd to compress digest layers |
 | `ENROOT_SQUASH_OPTIONS` | `-comp lzo -noD` | Options passed to mksquashfs to produce container images |
 | `ENROOT_MAX_PROCESSORS` | `$(nproc)` | Maximum number of processors to use for parallel tasks (0 means unlimited) |
 | `ENROOT_MAX_CONNECTIONS` | `10` | Maximum number of concurrent connections (0 means unlimited) |

enroot.in

@@ -70,6 +70,7 @@ config::export ENROOT_DATA_PATH        "${XDG_DATA_HOME}/enroot"
 config::export ENROOT_TEMP_PATH        "${TMPDIR:-/tmp}"
 
 config::export ENROOT_GZIP_PROGRAM     "$(command -v pigz > /dev/null && echo pigz || echo gzip)"
+config::export ENROOT_ZSTD_OPTIONS     "-1"
 config::export ENROOT_SQUASH_OPTIONS   "-comp lzo -noD"
 config::export ENROOT_MAX_PROCESSORS   "$(nproc)"
 config::export ENROOT_MAX_CONNECTIONS  10

pkg/deb/control

@@ -18,6 +18,7 @@ Depends: ${shlibs:Depends}, ${misc:Depends},
  parallel,
  passwd,
  squashfs-tools,
+ zstd,
 # coreutils,
 # findutils,
 # grep,

pkg/rpm/SPECS/enroot.spec

@@ -18,7 +18,7 @@ Summary: Unprivileged container sandboxing utility
 Conflicts: enroot
 %endif
 Requires: bash >= 4.2, curl, gawk, jq >= 1.5, parallel, shadow-utils, squashfs-tools
-Requires: coreutils, grep, findutils, gzip, glibc-common, sed, tar, util-linux
+Requires: coreutils, grep, findutils, gzip, glibc-common, sed, tar, util-linux, zstd
 #Recommends: pv, pigz, ncurses
 #Suggests: libnvidia-container-tools, squashfuse, fuse-overlayfs
 %description

src/docker.sh

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+source "${ENROOT_LIBRARY_PATH}/common.sh"
+
 readonly token_dir="${ENROOT_CACHE_PATH}/.tokens.${EUID}"
 readonly creds_file="${ENROOT_CONFIG_PATH}/.credentials"
 
@@ -80,6 +82,36 @@ docker::_authenticate() {
     fi
 }
 
+docker::_download_extract() (
+    local -r digest="$1"; shift
+    local curl_args=("$@")
+    local tmpfile= checksum=
+
+    set -euo pipefail
+    shopt -s lastpipe
+    umask 037
+
+    [ -e "${ENROOT_CACHE_PATH}/${digest}" ] && exit 0
+
+    trap 'common::rmall "${tmpfile}" 2> /dev/null' EXIT
+    tmpfile=$(mktemp -p "${ENROOT_CACHE_PATH}" "${digest}.XXXXXXXXXX")
+
+    exec {stdout}>&1
+    {
+        curl "${curl_args[@]}" | tee "/proc/self/fd/${stdout}" \
+          | "${ENROOT_GZIP_PROGRAM}" -d -f -c \
+          | zstd -T"$(expr "${ENROOT_MAX_PROCESSORS}" / "${ENROOT_MAX_CONNECTIONS}" \| 1)" -q -f -o "${tmpfile}" ${ENROOT_ZSTD_OPTIONS}
+    } {stdout}>&1 | sha256sum | common::read -r checksum x
+    exec {stdout}>&-
+
+    if [ "${digest}" != "${checksum}" ]; then
+        printf "Checksum mismatch: %s\n" "${digest}" >&2
+        exit 1
+    fi
+
+    mv -n "${tmpfile}" "${ENROOT_CACHE_PATH}/${digest}"
+)
+
 docker::_download() {
     local -r user="$1" registry="${2:-registry-1.docker.io}" tag="${4:-latest}" arch="$5"
     local image="$3"
@@ -133,20 +165,14 @@ docker::_download() {
           | readarray -t missing_digests
     fi
 
-    # Download digests, verify their checksums and put them in cache.
+    # Download digests, verify their checksums and extract them in the cache.
     if [ "${#missing_digests[@]}" -gt 0 ]; then
-        common::log INFO "Downloading ${#missing_digests[@]} missing digests..." NL
-        parallel --plain ${TTY_ON+--bar} --retries 3 -j "${ENROOT_MAX_CONNECTIONS}" -q curl "${curl_opts[@]}" -f -o {} "${req_params[@]}" -- \
-          "${url_digest}sha256:{}" ::: "${missing_digests[@]}"
-        common::log
-
-        common::log INFO "Validating digest checksums..." NL
-        parallel --plain -j "${ENROOT_MAX_PROCESSORS}" 'sha256sum -c <<< "{} {}"' ::: "${missing_digests[@]}" >&2
+        common::log INFO "Downloading ${#missing_digests[@]} missing layers..." NL
+        BASH_ENV="${BASH_SOURCE[0]}" parallel --plain ${TTY_ON+--bar} --shuf --retries 2 -j "${ENROOT_MAX_CONNECTIONS}" -q \
+          docker::_download_extract "{}" "${curl_opts[@]}" -f "${req_params[@]}" -- "${url_digest}sha256:{}" ::: "${missing_digests[@]}"
         common::log
-        chmod 640 "${missing_digests[@]}"
-        mv -n "${missing_digests[@]}" "${ENROOT_CACHE_PATH}"
     else
-        common::log INFO "Found all digests in cache"
+        common::log INFO "Found all layers in cache"
     fi
 
     # Return the container configuration along with all the layers.
@@ -216,7 +242,7 @@ docker::import() (
     local filename="$2" arch="$3"
     local layers=() config= image= registry= tag= user= tmpdir=
 
-    common::checkcmd curl grep awk jq parallel tar "${ENROOT_GZIP_PROGRAM}" find mksquashfs
+    common::checkcmd curl grep awk jq parallel tar "${ENROOT_GZIP_PROGRAM}" find mksquashfs zstd
 
     # Parse the image reference of the form 'docker://[<user>@][<registry>#]<image>[:<tag>]'.
     local -r reg_user="[[:alnum:]_.!~*\'()%\;:\&=+$,-@]+"
@@ -273,7 +299,7 @@ docker::import() (
     # Extract all the layers locally.
     common::log INFO "Extracting image layers..." NL
     parallel --plain ${TTY_ON+--bar} -j "${ENROOT_MAX_PROCESSORS}" mkdir {\#}\; tar -C {\#} --warning=no-timestamp --anchored --exclude='dev/*' \
-      --use-compress-program=\'"${ENROOT_GZIP_PROGRAM}"\' -pxf \'"${ENROOT_CACHE_PATH}/{}"\' ::: "${layers[@]}"
+      --use-compress-program=zstd -pxf \'"${ENROOT_CACHE_PATH}/{}"\' ::: "${layers[@]}"
     common::fixperms .
     common::log
 
@@ -284,7 +310,8 @@ docker::import() (
 
     # Configure the rootfs.
     mkdir 0
-    docker::configure "${PWD}/0" "${ENROOT_CACHE_PATH}/${config}" "${arch}"
+    zstd -q -d -o config "${ENROOT_CACHE_PATH}/${config}"
+    docker::configure "${PWD}/0" config "${arch}"
 
     # Create the final squashfs filesystem by overlaying all the layers.
     common::log INFO "Creating squashfs filesystem..." NL