tests script for board (#28)

* add marker for tests when running on board Signed-off-by: Thing-han, Lim <[email protected]> * add python script for running tests on board Signed-off-by: Thing-han, Lim <[email protected]> * use the python tests script in ci instead Signed-off-by: Thing-han, Lim <[email protected]> * clean up the original ci test script Signed-off-by: Thing-han, Lim <[email protected]> * move serial_marker macro into hal.h Signed-off-by: Thing-han, Lim <[email protected]> * -t -> -i to consist with pqm4 and import sys Signed-off-by: Thing-han, Lim <[email protected]> * update readme for the tests script usage Signed-off-by: Thing-han, Lim <[email protected]> --------- Signed-off-by: Thing-han, Lim <[email protected]>
pq-code-package · Apr 29, 2024 · 544e38a · 544e38a
1 parent 9bde5c5
commit 544e38a
Show file tree

Hide file tree

Showing 10 changed files with 354 additions and 155 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -47,15 +47,15 @@ jobs:
       id: func_test
       shell: nix develop .#ci -c bash -e {0}
       run: |
-          tests func
+          tests func -e -v
     - name: Speed test
       id: speed_test
       shell: nix develop .#ci -c bash -e {0}
       if: |
         success()
         || steps.func_test.conclusion == 'failure'
       run: |
-          tests speed
+          tests speed -e -v
     - name: Stack test
       id: stack_test
       shell: nix develop .#ci -c bash -e {0}
@@ -64,7 +64,7 @@ jobs:
         || steps.func_test.conclusion == 'failure'
         || steps.speed_test.conclusion == 'failure'
       run: |
-          tests stack
+          tests stack -e -v
     - name: Nistkat test
       shell: nix develop .#ci -c bash -e {0}
       if: |
@@ -74,4 +74,4 @@ jobs:
         || steps.stack_test.conclusion == 'failure'
       run: |
         make clean
-        tests nistkat
+        tests nistkat -e -v
diff --git a/README.md b/README.md
@@ -85,6 +85,7 @@ For example,
 - `make clean` cleans up intermediate artifacts
 - `make distclean` additionally cleanup the `libopencm3` library
 
+### Manual testing on board
 After generating the specified hex files, you can flash it to the development board using `openocd`. 
 For example,
 ```
@@ -95,3 +96,37 @@ To receive output from the develop board, you can, for example, use `pyserial-mi
 ```
     pyserial-miniterm /dev/<tty_device> 38400
 ```
+
+### Usage of the [tests script](scripts/tests)
+Make sure to run `make clean` between running tests on QEMU or on board or running func/stack/speed and nistkat tests. In case of any inconsistencies, refer to the help command for the most up-to-date usage information
+
+```
+▶ tests --help
+Usage: tests [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  func     Run functional tests
+  nistkat  Run nistkat tests
+  run      Run for the specified platform and hex file without parsing the
+           output
+  speed    Run speed tests
+  stack    Run stack tests
+```
+
+func/speed tests depends on the iteration parameter, which is passed to the tests in compile time, therefore it is preferred to build the binaries with the tests script
+
+```
+▶ tests func --help
+Usage: tests func [OPTIONS]
+
+Options:
+  -cfg, --platform-cfg PATH  Configuration file of the specified platform
+                             [default: hal/stm32f4discovery.cfg]
+  -v, --verbose              Show verbose output or not
+  -e, --emulate              Emulate on the QEMU or not
+  -i, --iterations INTEGER   Number of tests  [default: 1]
+  --help                     Show this message and exit.
+```
diff --git a/flake.nix b/flake.nix
@@ -28,6 +28,10 @@
             gcc-arm-embedded-13 # arm-gnu-toolchain-13.2.rel1
             qemu # 8.1.5
             yq
+
+            python311
+            python311Packages.pyserial # 3.5
+            python311Packages.click
           ];
         in
         {
@@ -38,7 +42,6 @@
 
               # debug dependencies
               openocd # 0.12.0
-              python311Packages.pyserial # 3.5
             ];
 
             shellHook = ''
@@ -50,7 +53,7 @@
             packages = core;
 
             shellHook = ''
-              export PATH=$PWD/scripts/ci:$PATH
+              export PATH=$PWD/scripts:$PWD/scripts/ci:$PATH
             '';
           };
         };

diff --git a/hal/hal.h b/hal/hal.h
@@ -5,6 +5,14 @@
 #include <stdint.h>
 #include <stdlib.h>
 
+#if !defined(MPS2_AN386)
+#define SERIAL_MARKER() {\
+        hal_send_str("$");\
+    }
+#else
+#define SERIAL_MARKER()
+#endif
+
 enum clock_mode {
     CLOCK_FAST,
     CLOCK_BENCHMARK

diff --git a/scripts/ci/tests b/scripts/ci/tests