From 544e38ab9cfc0c120f0e8794fb70644bbcd1d9cf Mon Sep 17 00:00:00 2001 From: "Lim, Thing-han" <15379156+potsrevennil@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:32:47 +0800 Subject: [PATCH] tests script for board (#28) * add marker for tests when running on board Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> * add python script for running tests on board Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> * use the python tests script in ci instead Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> * clean up the original ci test script Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> * move serial_marker macro into hal.h Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> * -t -> -i to consist with pqm4 and import sys Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> * update readme for the tests script usage Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> --------- Signed-off-by: Thing-han, Lim <15379156+potsrevennil@users.noreply.github.com> --- .github/workflows/build.yaml | 8 +- README.md | 35 +++++ flake.nix | 7 +- hal/hal.h | 8 + scripts/ci/tests | 143 ----------------- scripts/tests | 291 +++++++++++++++++++++++++++++++++++ test/nistkat.c | 3 + test/speed.c | 4 +- test/stack.c | 4 +- test/test.c | 6 +- 10 files changed, 354 insertions(+), 155 deletions(-) delete mode 100755 scripts/ci/tests create mode 100755 scripts/tests diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index da0095b..572b1d0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -47,7 +47,7 @@ jobs: id: func_test shell: nix develop .#ci -c bash -e {0} run: | - tests func + tests func -e -v - name: Speed test id: speed_test shell: nix develop .#ci -c bash -e {0} @@ -55,7 +55,7 @@ jobs: success() || steps.func_test.conclusion == 'failure' run: | - tests speed + tests speed -e -v - name: Stack test id: stack_test shell: nix develop .#ci -c bash -e {0} @@ -64,7 +64,7 @@ jobs: || steps.func_test.conclusion == 'failure' || steps.speed_test.conclusion == 'failure' run: | - tests stack + tests stack -e -v - name: Nistkat test shell: nix develop .#ci -c bash -e {0} if: | @@ -74,4 +74,4 @@ jobs: || steps.stack_test.conclusion == 'failure' run: | make clean - tests nistkat + tests nistkat -e -v diff --git a/README.md b/README.md index 748263a..5f1227a 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ For example, - `make clean` cleans up intermediate artifacts - `make distclean` additionally cleanup the `libopencm3` library +### Manual testing on board After generating the specified hex files, you can flash it to the development board using `openocd`. For example, ``` @@ -95,3 +96,37 @@ To receive output from the develop board, you can, for example, use `pyserial-mi ``` pyserial-miniterm /dev/ 38400 ``` + +### Usage of the [tests script](scripts/tests) +Make sure to run `make clean` between running tests on QEMU or on board or running func/stack/speed and nistkat tests. In case of any inconsistencies, refer to the help command for the most up-to-date usage information + +``` +▶ tests --help +Usage: tests [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + func Run functional tests + nistkat Run nistkat tests + run Run for the specified platform and hex file without parsing the + output + speed Run speed tests + stack Run stack tests +``` + +func/speed tests depends on the iteration parameter, which is passed to the tests in compile time, therefore it is preferred to build the binaries with the tests script + +``` +▶ tests func --help +Usage: tests func [OPTIONS] + +Options: + -cfg, --platform-cfg PATH Configuration file of the specified platform + [default: hal/stm32f4discovery.cfg] + -v, --verbose Show verbose output or not + -e, --emulate Emulate on the QEMU or not + -i, --iterations INTEGER Number of tests [default: 1] + --help Show this message and exit. +``` diff --git a/flake.nix b/flake.nix index f67fcdf..7ee8784 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,10 @@ gcc-arm-embedded-13 # arm-gnu-toolchain-13.2.rel1 qemu # 8.1.5 yq + + python311 + python311Packages.pyserial # 3.5 + python311Packages.click ]; in { @@ -38,7 +42,6 @@ # debug dependencies openocd # 0.12.0 - python311Packages.pyserial # 3.5 ]; shellHook = '' @@ -50,7 +53,7 @@ packages = core; shellHook = '' - export PATH=$PWD/scripts/ci:$PATH + export PATH=$PWD/scripts:$PWD/scripts/ci:$PATH ''; }; }; diff --git a/hal/hal.h b/hal/hal.h index f9b1c40..e5c7212 100644 --- a/hal/hal.h +++ b/hal/hal.h @@ -5,6 +5,14 @@ #include #include +#if !defined(MPS2_AN386) +#define SERIAL_MARKER() {\ + hal_send_str("$");\ + } +#else +#define SERIAL_MARKER() +#endif + enum clock_mode { CLOCK_FAST, CLOCK_BENCHMARK diff --git a/scripts/ci/tests b/scripts/ci/tests deleted file mode 100755 index 55190c2..0000000 --- a/scripts/ci/tests +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 - -set -o errexit -set -o errtrace -set -o nounset -set -o pipefail - -ROOT="$(realpath "$(dirname "$0")"/../../)" -SCHEMES=(mlkem512 mlkem768 mlkem1024) - -base_test() -{ - local TEST_TYPE="$1" - local tests="${2:-10}" - local expect_proc="$3" - local actual_proc="$4" - - success=true - - if ! ls "$ROOT/elf/"*"$TEST_TYPE.elf" >/dev/null 2>&1; then - output=$(make "emulate $TEST_TYPE") - if [[ $? == 1 ]]; then - echo "::error file={$file},title={Build failed}::$output" 1>&2 - exit 1 - fi - fi - - for SCHEME in "${SCHEMES[@]}"; do - file="$ROOT/elf/$SCHEME-$TEST_TYPE.elf" - - # build and emulate - output=$(make "emulate run" ELF_FILE="$file" && echo -n "x") && true - if [[ $? == 1 ]]; then - echo "::error file={$file},title={Emulate failed}::$output" 1>&2 - success=false - continue - fi - - output="${output%?}" - - # check for error - err=$(echo "$output" | awk '/ERROR/{code=1;print $0} END {exit code}') && true - if [[ $? == 1 ]]; then - echo "::error file={$file},title={Emulate unexpected}::$err" 1>&2 - success=false - fi - - # check if is as expected - actual=$(echo -n "$output" | "$actual_proc") - expect=$("$expect_proc" "$SCHEME") - - if [[ $actual != "$expect" ]]; then - echo "::error file={$file},title={Emulate unexpected}::Expected $expect, but received $actual" 1>&2 - success=false - else - echo "$file emulate as expected" - fi - done - - if ! "$success"; then - exit 1 - fi -} - -cal_count() -{ - awk 'BEGIN{c=0} /OK/ {c++} END {print c}' -} - -cal_sha256() -{ - sha256sum | awk '{ print $1 }' -} - -func() -{ - local TEST_TYPE="test" - local tests=${1:-10} - local expect=$((tests * 3)) - ( - expect_proc() - { - echo $((tests * 3)) - } - - base_test "$TEST_TYPE" "$tests" expect_proc cal_count - ) -} - -speed() -{ - local TEST_TYPE="speed" - local tests=${1:-10} - ( - expect_proc() - { - echo "$tests" - } - - base_test "$TEST_TYPE" "$tests" expect_proc cal_count - ) -} - -stack() -{ - local TEST_TYPE="stack" - ( - expect_proc() - { - echo "1" - } - - base_test "$TEST_TYPE" "" expect_proc cal_count - ) -} - -nistkat() -{ - local TEST_TYPE="nistkat" - ( - expect_proc() - { - local out - out=$(yq -r --arg scheme "$1" ' - .implementations.[] - | select(.name == $scheme) - | ."nistkat-sha256"' "$ROOT/META.yml") - echo "$out" - } - - base_test "$TEST_TYPE" "" expect_proc cal_sha256 - ) -} - -declare -f -F "$1" && true >/dev/null - -if [[ $? == 1 ]]; then - echo "Function not exists" 1>&2 - exit 2 -fi - -"$@" diff --git a/scripts/tests b/scripts/tests new file mode 100755 index 0000000..b345f11 --- /dev/null +++ b/scripts/tests @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: Apache-2.0 +import serial +import hashlib +import platform +import glob +import asyncio +import subprocess +import click +import logging +import sys +from functools import reduce + +# utilities for interacting with the board + + +def serial_ports(): + """Lists serial port names""" + if platform.system() == "Windows": + ports = ["COM%s" % (i + 1) for i in range(256)] + elif platform.system() == "Darwin": + ports = glob.glob("/dev/tty.usbserial*") + elif platform.system() == "Linux": + ports = glob.glob("/dev/ttyUSB*") + else: + raise PlatformError("Unsupported platform") + + return ports + + +def usbdev(): + port = next(iter(serial_ports()), None) + + if port is None: + raise DeviceError("No available usb device") + + dev = serial.Serial(port, 38400) + return dev + + +async def flash(cfg, bin): + subprocess.run( + ["openocd", "-f", cfg, "-c", f"program {bin} verify reset exit"], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + + +async def read(dev): + logging.debug("Waiting for start marker") + x = dev.read_until(b"$") + logging.debug("Reading input") + x = dev.read_until(b"$") + return x[1:-1] + + +async def flush_and_read(dev, cfg, bin): + # assume the first result is the usb device to use + asyncio.create_task(flash(cfg, bin)) + read_task = asyncio.create_task(read(dev)) + result = await read_task + logging.debug(str(result, encoding="utf-8")) + return result + + +# test utilities + + +def config_logger(verbose): + if verbose: + logging.basicConfig(format="%(levelname)-5s > %(message)s", level=logging.DEBUG) + else: + logging.basicConfig(format="%(levelname)-5s > %(message)s", level=logging.INFO) + + +def count(bstr, match): + return str(bstr, encoding="utf8").count(match) + + +def base_test( + test_type, emulate, platform_cfg, ntests, expect_proc, actual_proc, verbose +): + """ + test_type: test, speed, stack, nistkat + expect_proc: scheme -> any + actual_proc: str -> any + """ + config_logger(verbose) + + if emulate: + subprocess.run( + ["make", f"NTESTS={ntests}", f"emulate {test_type}"], + stdout=subprocess.DEVNULL, + ) + else: + subprocess.run( + ["make", f"{test_type}"] + + [f"NTESTS={ntests}"] * (test_type != "nistkat") + + ["KATRNG=NIST"] * (test_type == "nistkat"), + stdout=subprocess.DEVNULL, + ) + + def check(file, expect, actual): + if actual != expect: + logging.error( + f"Check {file} failed, expecting {expect}, but getting actual {actual}" + ) + sys.exit(1) + else: + logging.info(f"Check {file} passed") + + if not emulate: + dev = usbdev() + + for scheme in ["mlkem512", "mlkem768", "mlkem1024"]: + file = f"elf/{scheme}-{test_type}.elf" + expect = expect_proc(scheme) + if emulate: + actual = actual_proc( + subprocess.run( + ["make", "emulate run", f"ELF_FILE={file}"], capture_output=True + ).stdout + ) + else: + actual = actual_proc(asyncio.run(flush_and_read(dev, platform_cfg, file))) + + check(file, expect, actual) + + +# cli utilities + +_shared_options = [ + click.option( + "-cfg", + "--platform-cfg", + default="hal/stm32f4discovery.cfg", + type=click.Path(), + help="Configuration file of the specified platform", + ), + click.option( + "-v", + "--verbose", + is_flag=True, + show_default=True, + default=False, + type=bool, + help="Show verbose output or not", + ), + click.option( + "-e", + "--emulate", + is_flag=True, + show_default=True, + default=False, + type=bool, + help="Emulate on the QEMU or not", + ), +] + + +def add_options(options): + return lambda func: reduce(lambda f, o: o(f), reversed(options), func) + + +# commands + + +@click.command( + short_help="Run for the specified platform and hex file without parsing the output", + context_settings={"show_default": True}, +) +@add_options(_shared_options) +@click.option( + "-b", + "--bin", + type=click.Path(), + help="The binary hex file that you wanted to test.", +) +def run(platform_cfg, bin, verbose, emulate): + config_logger(True) + dev = usbdev() + + try: + result = asyncio.run(flush_and_read(dev, platform_cfg, bin)) + logging.info(result) + except asyncio.CancelledError: + pass + + +@click.command( + short_help="Run functional tests", context_settings={"show_default": True} +) +@add_options(_shared_options) +@click.option("-i", "--iterations", default=1, type=int, help="Number of tests") +def func(platform_cfg, iterations, verbose, emulate): + try: + base_test( + "test", + emulate, + platform_cfg, + iterations, + lambda _: iterations * 3, + lambda output: count(output, "OK"), + verbose, + ) + except asyncio.CancelledError: + pass + + +@click.command(short_help="Run speed tests", context_settings={"show_default": True}) +@add_options(_shared_options) +@click.option("-i", "--iterations", default=1, type=int, help="Number of tests") +def speed(platform_cfg, iterations, verbose, emulate): + try: + base_test( + "speed", + emulate, + platform_cfg, + iterations, + lambda _: iterations, + lambda output: count(output, "OK"), + verbose, + ) + except asyncio.CancelledError: + pass + + +@click.command(short_help="Run stack tests", context_settings={"show_default": True}) +@add_options(_shared_options) +def stack(platform_cfg, verbose, emulate): + try: + base_test( + "stack", + emulate, + platform_cfg, + 0, + lambda _: 1, + lambda output: count(output, "OK"), + verbose, + ) + except asyncio.CancelledError: + pass + + +@click.command(short_help="Run nistkat tests", context_settings={"show_default": True}) +@add_options(_shared_options) +def nistkat(platform_cfg, verbose, emulate): + def scheme_hash(scheme): + result = subprocess.run( + [ + "yq", + "-r", + "--arg", + "scheme", + scheme, + '.implementations.[] | select(.name == $scheme) | ."nistkat-sha256"', + "./META.yml", + ], + capture_output=True, + encoding="utf-8", + universal_newlines=False, + ) + return result.stdout.strip() + + try: + base_test( + "nistkat", + emulate, + platform_cfg, + 0, + scheme_hash, + lambda output: hashlib.sha256(output).hexdigest(), + verbose, + ) + except asyncio.CancelledError: + pass + + +@click.group() +def cli(): + pass + + +cli.add_command(run) +cli.add_command(func) +cli.add_command(speed) +cli.add_command(stack) +cli.add_command(nistkat) + +if __name__ == "__main__": + cli() diff --git a/test/nistkat.c b/test/nistkat.c index 708fad1..74723ab 100644 --- a/test/nistkat.c +++ b/test/nistkat.c @@ -13,6 +13,7 @@ #include "hal.h" #include "randombytes.h" +// NOTE: used Kyber in the nistkat rsp file for now to avoid changing the checksum #if (MLKEM_K == 2) #define OLD_CRYPTO_ALGNAME "Kyber512" #elif (MLKEM_K == 3) @@ -62,6 +63,7 @@ int main(void) { int rc; hal_setup(CLOCK_FAST); + SERIAL_MARKER(); for (uint8_t i = 0; i < 48; i++) { entropy_input[i] = i; @@ -110,6 +112,7 @@ int main(void) { hal_send_str(""); } + SERIAL_MARKER(); return 0; diff --git a/test/speed.c b/test/speed.c index 05c02b2..9191ae2 100644 --- a/test/speed.c +++ b/test/speed.c @@ -18,7 +18,7 @@ int main(void) { hal_setup(CLOCK_BENCHMARK); - hal_send_str("=========================="); + SERIAL_MARKER(); for (i = 0; i < NTESTS; i++) { // Key-pair generation @@ -47,7 +47,7 @@ int main(void) { hal_send_str("+"); } - hal_send_str("#"); + SERIAL_MARKER(); return 0; } diff --git a/test/stack.c b/test/stack.c index 3d942d7..bde5269 100644 --- a/test/stack.c +++ b/test/stack.c @@ -70,10 +70,10 @@ int main(void) { hal_setup(CLOCK_FAST); // marker for automated benchmarks - hal_send_str("=========================="); + SERIAL_MARKER(); test_keys(); // marker for automated benchmarks - hal_send_str("#"); + SERIAL_MARKER(); return 0; } diff --git a/test/test.c b/test/test.c index 4a3c183..f2acbe9 100644 --- a/test/test.c +++ b/test/test.c @@ -144,11 +144,13 @@ int main(void) { hal_setup(CLOCK_FAST); // marker for automated testing - hal_send_str("=========================="); + SERIAL_MARKER(); + test_keys(); test_invalid_sk_a(); test_invalid_ciphertext(); - hal_send_str("#"); + + SERIAL_MARKER(); return 0; }