Filesystem benchmark based on fio.

To make a contrast, this benchmark will test all three types of image,
OCI, stargz and estargz in following steps.

1.Use fio to generate fake pread() requests parallely (4 threads).
2.Scrape metrics in /proc/<pid of stargz-snapshotter>.
3.Calculate scraped metrics with PromQL.
4.Draw fio bandwidth-latency and process-metrics images with gunplot.

Signed-off-by: Chuang Zhang <[email protected]>

Makefile

@@ -19,8 +19,8 @@ GO111MODULE_VALUE=auto
 PREFIX ?= out/
 
 CMD=containerd-stargz-grpc ctr-remote
-
 CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
+BENCHMARK_OUTPUT=out-bench/
 
 .PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-pullsecrets test-cri
 
@@ -61,6 +61,7 @@ uninstall:
 clean:
 	@echo "$@"
 	@rm -f $(CMD_BINARIES)
+	@rm -rf $(BENCHMARK_OUTPUT)
 
 test:
 	@echo "$@"
@@ -81,6 +82,9 @@ test-optimize:
 benchmark:
 	@./script/benchmark/test.sh
 
+fs-bench:
+	@./script/fs-bench/test.sh
+
 test-pullsecrets:
 	@./script/pullsecrets/test.sh
 

script/fs-bench/fio/Dockerfile

@@ -0,0 +1,38 @@
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM alpine:3.9 AS stage-build
+
+WORKDIR /work
+RUN apk add git build-base linux-headers && \
+    git clone git://git.kernel.dk/fio.git /work/fio && \
+    cd /work/fio && \
+    git checkout c96b385b6e0c78478697713e6da9174fba2432d3 && \
+    ./configure --prefix=/work/root && \
+    make && \
+    make install
+
+FROM alpine:3.9 AS stage-final
+
+# How many bytes fio is going to read/write, 512MiB by default.
+ARG size=256M
+
+# How many concurrent reading threads.
+ARG thread=4
+
+WORKDIR /work
+COPY --from=stage-build /work/root/ /
+
+# Lay out test files here, so that they can be compressed into the image layer.
+RUN fio -directory=/work -direct=1 -numjobs=$thread -size=$size -name=test -rw=read

script/fs-bench/test.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -euo pipefail
+
+cleanup() {
+    local ORG_EXIT_CODE="${1}"
+    rm "${DOCKER_COMPOSE_YAML}" || true
+    exit "${ORG_EXIT_CODE}"
+}
+trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
+
+CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/"
+REPO="${CONTEXT}../../"
+
+echo "Preparing docker-compose.yml..."
+DOCKER_COMPOSE_YAML=$(mktemp)
+BENCHMARKING_NODE=fs-bench
+BENCHMARKING_CONTAINER=fs-bench
+cat <<EOF > "${DOCKER_COMPOSE_YAML}"
+version: "3"
+services:
+  ${BENCHMARKING_NODE}:
+    build:
+      context: "${CONTEXT}/work"
+      dockerfile: Dockerfile
+    container_name: ${BENCHMARKING_CONTAINER}
+    privileged: true
+    working_dir: /go/src/github.com/containerd/stargz-snapshotter
+    command: tail -f /dev/null
+    environment:
+    - NO_PROXY=127.0.0.1,localhost
+    - HTTP_PROXY=${HTTP_PROXY:-}
+    - HTTPS_PROXY=${HTTPS_PROXY:-}
+    - http_proxy=${http_proxy:-}
+    - https_proxy=${https_proxy:-}
+    tmpfs:
+    - /tmp:exec,mode=777
+    volumes:
+    - "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro"
+    - "/dev/fuse:/dev/fuse"
+    - "containerd-data:/var/lib/containerd:delegated"
+    - "containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc:delegated"
+    - "containerd-stargz-grpc-status:/run/containerd-stargz-grpc:delegated"
+volumes:
+  containerd-data:
+  containerd-stargz-grpc-data:
+  containerd-stargz-grpc-status:
+EOF
+
+echo "Setup benchmark environment..."
+docker-compose -f "${DOCKER_COMPOSE_YAML}" build ${DOCKER_BUILD_ARGS:-} "${BENCHMARKING_NODE}"
+docker-compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate
+
+echo "Benchmarking..."
+docker exec -i "${BENCHMARKING_CONTAINER}" script/fs-bench/work/run.sh
+
+echo "Harvesting output..."
+docker cp "${BENCHMARKING_CONTAINER}:/output" "${REPO}/out-bench"
+
+echo "Cleaning up benchmark environment..."
+docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v

script/fs-bench/work/Dockerfile

@@ -0,0 +1,43 @@
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM golang:1.13
+
+# basic tools
+COPY ./tools /tmp/tools
+
+RUN apt-get update -y && \
+    apt-get install --no-install-recommends -y libbtrfs-dev libseccomp-dev fuse python \
+        apt-transport-https software-properties-common && \
+    curl -Lo gnuplot-5.2.8.tar.gz \
+    https://sourceforge.net/projects/gnuplot/files/gnuplot/5.2.8/gnuplot-5.2.8.tar.gz/download && \
+    tar xf gnuplot-5.2.8.tar.gz && \
+    cd gnuplot-5.2.8 && \
+    ./configure && make && make install && \
+    cd /tmp/tools && \
+    GO111MODULE=on go build -o "/usr/local/bin/process" "./process/main.go" && \
+    GO111MODULE=on go build -o "/usr/local/bin/scrape" "./scrape/main.go"
+
+# runtime dependencies
+RUN git clone https://github.com/opencontainers/runc \
+          $GOPATH/src/github.com/opencontainers/runc && \
+    cd $GOPATH/src/github.com/opencontainers/runc && \
+    git checkout d736ef14f0288d6993a1845745d6756cfc9ddd5a && \
+    GO111MODULE=off make -j2 BUILDTAGS='seccomp apparmor' && \
+    GO111MODULE=off make install && \
+    git clone https://github.com/containerd/containerd \
+          $GOPATH/src/github.com/containerd/containerd && \
+    cd $GOPATH/src/github.com/containerd/containerd && \
+    git checkout 990076b731ec9446437972b41176a6b0f3b7bcbf && \
+    GO111MODULE=off make -j2 && GO111MODULE=off make install

script/fs-bench/work/config/config.containerd.toml

@@ -0,0 +1,4 @@
+[proxy_plugins]
+  [proxy_plugins.stargz]
+    type = "snapshot"
+    address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"

script/fs-bench/work/config/config.stargz.toml

@@ -0,0 +1 @@
+noprefetch = true

script/fs-bench/work/config/fio.conf

@@ -0,0 +1,10 @@
+[test]
+# TODO randomize the size of each read (pread, to be precise).
+bs=4k
+rw=randread
+size=256M
+numjobs=4
+directory=/work
+write_bw_log=/output/test
+write_iops_log=/output/test
+write_lat_log=/output/test

script/fs-bench/work/reset.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -euo pipefail
+
+CONTAINERD_ROOT=/var/lib/containerd/
+CONTAINERD_CONFIG_DIR=/etc/containerd/
+REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-stargz-grpc/containerd-stargz-grpc.sock
+REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd-stargz-grpc/
+REMOTE_SNAPSHOTTER_CONFIG_DIR=/etc/containerd-stargz-grpc/
+
+for arg; do
+    case x"$arg" in
+    x-nosnapshotter)
+        NOSNAPSHOTTER="-nosnapshotter"
+        ;;
+    x-nocleanup)
+        NOCLEANUP="-nocleanup"
+        ;;
+    x*)
+        SCRAPE_OUTPUT_FILENAME="$arg"
+        ;;
+    esac
+done
+
+RETRYNUM=30
+RETRYINTERVAL=1
+TIMEOUTSEC=180
+function retry {
+    local SUCCESS=false
+    for i in $(seq ${RETRYNUM}) ; do
+        if eval "timeout ${TIMEOUTSEC} ${@}" ; then
+            SUCCESS=true
+            break
+        fi
+        echo "Fail(${i}). Retrying..."
+        sleep ${RETRYINTERVAL}
+    done
+    if [ "${SUCCESS}" == "true" ] ; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function kill_all {
+    if [ "${1}" != "" ] ; then
+        ps aux | grep "${1}" | grep -v grep | grep -v $(basename ${0}) | sed -E 's/ +/ /g' | cut -f 2 -d ' ' | xargs -I{} kill -9 {} || true
+    fi
+}
+
+function cleanup {
+    rm -rf "${CONTAINERD_ROOT}"*
+    if [ -f "${REMOTE_SNAPSHOTTER_SOCKET}" ] ; then
+        rm "${REMOTE_SNAPSHOTTER_SOCKET}"
+    fi
+    if [ -d "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" ] ; then 
+        find "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" \
+             -maxdepth 1 -mindepth 1 -type d -exec umount "{}/fs" \;
+    fi
+    rm -rf "${REMOTE_SNAPSHOTTER_ROOT}"*
+}
+
+echo "cleaning up the environment..."
+kill_all "containerd"
+kill_all "containerd-stargz-grpc"
+kill_all "scrape"
+if [ "$NOCLEANUP" == "-nocleanup" ]; then
+    echo "DO NOT CLEANUP containerd & stargz-snapshotter layers"
+else
+    cleanup
+fi
+if [ "${NOSNAPSHOTTER}" == "-nosnapshotter" ] ; then
+    echo "DO NOT RUN remote snapshotter"
+else
+    echo "running remote snaphsotter..."
+    containerd-stargz-grpc --log-level=debug \
+        --address="${REMOTE_SNAPSHOTTER_SOCKET}" \
+        --config="${REMOTE_SNAPSHOTTER_CONFIG_DIR}config.stargz.toml" \
+        &>/var/log/containerd-stargz-grpc.log &
+    retry ls "${REMOTE_SNAPSHOTTER_SOCKET}"
+    mkdir -p "$(dirname "${SCRAPE_OUTPUT_FILENAME}")"
+    scrape -output "${SCRAPE_OUTPUT_FILENAME}" -netinf eth0 -interval 1s \
+        -pid "$(ps aux | grep -v grep | grep containerd-stargz-grpc | awk '{print $2}')" &
+fi
+echo "running containerd..."
+containerd --log-level debug \
+    --config="${CONTAINERD_CONFIG_DIR}config.containerd.toml" &>/var/log/containerd.log &
+retry ctr version