Merge pull request #7 from mwasilew/remove_ltp

test: remove LTP

Dockerfile

@@ -1,19 +1,3 @@
-# Build LTP ------------------------------------------------------------------
-FROM ubuntu:20.04 as ltp
-
-RUN apt update && \
-	apt install -y gcc git make pkgconf autoconf automake bison flex m4 libc6-dev wget
-
-RUN wget -O /ltp.tar.xz https://github.com/linux-test-project/ltp/releases/download/20200515/ltp-full-20200515.tar.xz
-RUN tar -xf /ltp.tar.xz
-RUN	cd ltp-full* && \
-	./configure && \
-	make -j8 all && \
-	make install
-
-RUN cd /opt/ltp/testcases/bin && \
-	(strip `ls | grep -v .sh | grep -v .py` || true)
-
 # Build Python Deps------------------------------------------------------------
 FROM ubuntu:20.04 as pydeps
 
@@ -37,7 +21,6 @@ RUN \
 	ln -s /usr/lib/*-linux-gnu/engines-1.1 /usr/lib/engines-1.1 && \
 	rm -rf /var/lib/apt/lists/*
 
-COPY --from=ltp /opt/ltp /opt/ltp
 COPY --from=pydeps /usr/local/lib/python3.8/dist-packages/ /usr/local/lib/python3.8/dist-packages/
 COPY ./bin/* /usr/local/bin/
 COPY ./tests /usr/share/fio-tests

README.md

@@ -29,53 +29,53 @@ to send callback messages to it. The container then listens for 2 callbacks:
 A testing specification allows a user to define how target testing should
 take place. Some users may want to do a single set of tests like:
 ~~~
-# this runs the LTP test suite one time after a Target is installed
+# this runs the smoke test suite one time after a Target is installed
 sequence:
   - tests:
-      - name: ltp
+      - name: smoke test
         command:
-          - /usr/share/fio-tests/ltp.sh
+          - /usr/share/fio-tests/smoke.sh
 ~~~
 
 Some users may want to do repeated "soak" testing of a device:
 ~~~
-# run LTP test suite over and over taking a 10 minute break in between tests
+# run smoke test suite over and over taking a 10 minute break in between tests
 sequence:
   - tests:
-      - name: ltp
+      - name: smoke test
         command:
-          - /usr/share/fio-tests/ltp.sh
+          - /usr/share/fio-tests/smoke.sh
     repeat:
       delay_seconds: 600
 ~~~
 
 A test can also be repeated a specific number of times:
 ~~~
-# run LTP test suite 4 times with a 5 second delay between runs
+# run smoke test suite 4 times with a 5 second delay between runs
 sequence:
   - tests:
-      - name: ltp
+      - name: smoke test
         command:
-          - /usr/share/fio-tests/ltp.sh
+          - /usr/share/fio-tests/smoke.sh
     repeat:
       delay_seconds: 5
       total: 4
 ~~~
 
 A test can even force the device to reboot with:
 ~~~
-# Run the LTP suite, reboot, and run it again
+# Run the smoke suite, reboot, and run it again
 sequence:
   - tests:
-      - name: ltp
+      - name: smoke test
         command:
-          - /usr/share/fio-tests/ltp.sh
+          - /usr/share/fio-tests/smoke.sh
   - reboot:
       command: /sbin/reboot
   - tests:
-      - name: ltp
+      - name: smoke test
         command:
-          - /usr/share/fio-tests/ltp.sh
+          - /usr/share/fio-tests/smoke.sh
 ~~~
 
 A test can be run directly on the host with:
@@ -95,3 +95,82 @@ sequence:
    hub.foundries.io/lmp/fiotest might be needed for customized tests.
 2. Create a custom test-spec.yml.
 3. Update docker-compose.yml to reference 1 and 2.
+
+### Example extension - add LTP test suite
+
+#### Create LTP execution script - tests/ltp.sh
+~~~
+#!/bin/bash -e
+
+set -o pipefail
+
+TESTS="${TESTS-syscalls -s madvise}"
+LTP_PATH="${LTP_PATH-/opt/ltp}"
+LOGS="/tmp/LTP_$(date +%s)_"
+
+[ -z $TEST_DIR ] && (echo "ERROR: TEST_DIR not defined"; exit 1)
+[ -d $TEST_DIR ] || (echo "ERROR: no TEST_DIR"; exit 1)
+
+read_ltp_results() {
+	grep -E "PASS|FAIL|CONF"  "$1" \
+		| awk '{print $1" "$2}'
+}
+
+failed=0
+run_ltp() {
+	cd "${LTP_PATH}"
+
+	./runltp -p -q -f $TESTS -l "${LOGS}.log" 2>&1 | tee ${LOGS}-output.log || true
+	while IFS= read -r line ; do
+		parts=($line)
+		result_dir="${TEST_DIR}/$(date +%s.%N)-${parts[0]}"
+		mkdir ${result_dir}
+		if [ "${parts[1]}" = "FAIL" ] ; then
+			touch ${result_dir}/failed
+			failed=1
+		fi
+		if [ "${parts[1]}" = "CONF" ] ; then
+			touch ${result_dir}/skipped
+		fi
+	done < <(read_ltp_results "${LOGS}.log")
+}
+
+run_ltp
+exit $failed
+~~~
+
+#### Create Dockerfile extending fiotest
+~~~
+# Build LTP
+FROM ubuntu:20.04 as ltp
+
+RUN apt update && \
+       apt install -y gcc git make pkgconf autoconf automake bison flex m4 libc6-dev wget
+
+RUN wget -O /ltp.tar.xz https://github.com/linux-test-project/ltp/releases/download/20200515/ltp-full-20200515.tar.xz
+RUN tar -xf /ltp.tar.xz
+RUN    cd ltp-full* && \
+       ./configure && \
+       make -j8 all && \
+       make install
+
+RUN cd /opt/ltp/testcases/bin && \
+       (strip `ls | grep -v .sh | grep -v .py` || true)
+
+# Extend fiotest
+FROM hub.foundries.io/lmp/fiotest:postmerge
+
+COPY --from=ltp /opt/ltp /opt/ltp
+COPY ./tests/ltp.sh /usr/share/fio-tests/ltp.sh
+~~~
+
+#### Add LTP to test-spec.yaml
+~~~
+sequence:
+    tests:
+...
+      - name: ltp
+         command:
+          - /usr/share/fio-tests/ltp.sh
+...
+~~~

test-spec.yml

@@ -1,9 +1,22 @@
 sequence:
   - tests:
-      - name: ltp
+      - name: block devices
         command:
-          - /usr/share/fio-tests/ltp.sh
-    # repeat not specified, so runs one time
+          - /usr/bin/lsblk
+        on_host: true
+      - name: cpus
+        command:
+          - /usr/bin/lscpu
+        on_host: true
+      - name: usb devices
+        command:
+          - /usr/bin/lsusb
+        on_host: true
+      - name: smoke tests
+        command:
+          - /usr/share/fio-tests/smoke.sh
+        on_host: true
+
   - reboot:
       command:
         - /bin/true

tests/smoke.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+
+set -o pipefail
+
+TESTS="pwd, uname -a, lscpu, vmstat, lsblk"
+
+[ -z $TEST_DIR ] && (echo "ERROR: TEST_DIR not defined"; exit 1)
+[ -d $TEST_DIR ] || (echo "ERROR: no TEST_DIR"; exit 1)
+
+indent() {
+    sed 's/^/|  /'
+    echo "|--"
+}
+
+run() {
+    # shellcheck disable=SC2039
+    local test="$1"
+    test_case_id="$(echo "${test}" | awk '{print $1}')"
+    result_dir="${TEST_DIR}/$(date +%s.%N)-${test_case_id}"
+    mkdir ${result_dir}
+    echo
+    echo "Running ${test_case_id} test..."
+    local exit_code=0
+    eval "${test}" | indent || exit_code="$?"
+    echo "Exit code: $exit_code"
+
+    if [ "${exit_code}" -ne 0 ]; then
+        touch ${result_dir}/failed
+    fi
+    return "${exit_code}"
+}
+
+failed=0
+IFS=","
+for test_cmd in $TESTS ; do
+    return_code=0
+    run "${test_cmd}" || return_code="$?"
+    echo "Return code: $return_code"
+    if [ "$return_code" -ne 0 ]; then
+        failed=1
+    fi
+    TESTS="$(echo "${TESTS}" | sed -r "s#${test_cmd},? *##")"
+done
+exit $failed