From 029a3d1bbabb4808c46ef2c15c0e7e7b4543cd37 Mon Sep 17 00:00:00 2001 From: Gregor Haas Date: Mon, 24 Jun 2024 15:55:50 -0700 Subject: [PATCH] Implement CI for 64-bit CVA6 (#452) This PR extends our CI system to also run tests on real CVA6 hardware, to ensure that we don't inadvertently break this platform. The CI infrastructure depends on a bitstream version that includes our changes to the bootrom (https://github.com/openhwgroup/cva6/pull/2267), which implement a simple UART-based firmware flashing functionality. As part of extending our CI to this platform, we spent some effort to make the system hostable and extensible by others. We check in some previously unofficial helper scripts that operate an FT245R relay board, enabling us to turn boards off and on remotely. We also include some infrastructure for discovering serial ports (since our runner machine has many of these). --- .github/workflows/main.yml | 4 +- .github/workflows/test-system.yml | 207 ++++-------------- .gitmodules | 3 + mkutils/plat/cva6/run.mk | 24 +- mkutils/plat/mpfs/run.mk | 2 +- .../board/cva6/configs/linux64-cva6-defconfig | 25 +-- .../board/cva6/configs/uboot-cva6.config | 4 + overlays/keystone/board/cva6/post-image.sh | 19 ++ .../keystone/configs/riscv64_cva6_defconfig | 12 +- scripts/ci/configs/global.sh | 13 ++ scripts/ci/configs/track.sh | 22 ++ .../cva6/expected.log} | 1 - scripts/ci/plat/cva6/flash-firmware.sh | 59 +++++ scripts/ci/plat/cva6/flash-os.sh | 43 ++++ scripts/ci/plat/cva6/test.sh | 48 ++++ scripts/ci/{ => plat/generic}/expected.log | 1 - scripts/ci/plat/generic/test.sh | 40 ++++ scripts/ci/plat/mpfs/expected.log | 17 ++ scripts/ci/plat/mpfs/flash-firmware.sh | 31 +++ scripts/ci/plat/mpfs/flash-os.sh | 45 ++++ scripts/ci/plat/mpfs/test.sh | 44 ++++ scripts/ci/test-setup.sh | 138 ++++++++++++ scripts/ci/utils/find_tty.sh | 29 +++ scripts/ci/utils/relay_ft245r | 1 + scripts/ci/utils/relay_power.py | 58 +++++ scripts/ci/utils/wait_for.py | 27 +++ sm/src/platform/fpga/ariane/platform.h | 2 +- 27 files changed, 731 insertions(+), 188 deletions(-) create mode 100644 overlays/keystone/board/cva6/configs/uboot-cva6.config create mode 100755 overlays/keystone/board/cva6/post-image.sh create mode 100644 scripts/ci/configs/global.sh create mode 100644 scripts/ci/configs/track.sh rename scripts/ci/{expected-mpfs.log => plat/cva6/expected.log} (99%) create mode 100755 scripts/ci/plat/cva6/flash-firmware.sh create mode 100755 scripts/ci/plat/cva6/flash-os.sh create mode 100755 scripts/ci/plat/cva6/test.sh rename scripts/ci/{ => plat/generic}/expected.log (99%) create mode 100755 scripts/ci/plat/generic/test.sh create mode 100644 scripts/ci/plat/mpfs/expected.log create mode 100755 scripts/ci/plat/mpfs/flash-firmware.sh create mode 100755 scripts/ci/plat/mpfs/flash-os.sh create mode 100755 scripts/ci/plat/mpfs/test.sh create mode 100755 scripts/ci/test-setup.sh create mode 100755 scripts/ci/utils/find_tty.sh create mode 160000 scripts/ci/utils/relay_ft245r create mode 100755 scripts/ci/utils/relay_power.py create mode 100755 scripts/ci/utils/wait_for.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe16e9327..445c0d5cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -222,7 +222,7 @@ jobs: # Combine cache directories to save space combine-caches: runs-on: ubuntu-latest - if: ${{ always() }} + if: success() || failure() needs: build steps: - name: Install dependencies @@ -378,6 +378,6 @@ jobs: uses: ./.github/workflows/build-runtime.yml # System tests, which are run for simulatable and self-hostable platforms - test-system: + test-system-functionality: needs: build uses: ./.github/workflows/test-system.yml diff --git a/.github/workflows/test-system.yml b/.github/workflows/test-system.yml index 11f7f5592..43a8650cd 100644 --- a/.github/workflows/test-system.yml +++ b/.github/workflows/test-system.yml @@ -3,19 +3,31 @@ on: workflow_call: jobs: - test-generic: - runs-on: ubuntu-latest + test-system: + runs-on: ${{ matrix.platform == 'generic' && 'ubuntu-latest' || matrix.platform }} + environment: ${{ matrix.platform != 'generic' && 'track' || null }} strategy: + fail-fast: false matrix: - platform: [generic] + platform: [generic, mpfs, cva6] bits: [32, 64] + exclude: + # mpfs is not 32 bit + - platform: mpfs + bits: 32 + # ignore 32-bit cva6 for now + - platform: cva6 + bits: 32 steps: - # We don't need submodules here since Keystone is a monorepo! - name: Checkout Keystone uses: actions/checkout@v4 with: - submodules: 'false' + submodules: 'true' + sparse-checkout: | + . + scripts/ + mkutils/ - name: Restore build directory uses: actions/download-artifact@v4 @@ -26,188 +38,63 @@ jobs: - name: Decompress build directory run: cat build.tar.xz | xz -d -T0 | tar -xf - - - name: Test Keystone system - run: | - # Fix permissions on the key - chmod 600 build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/target/root/.ssh/id-rsa - - # Launch QEMU - export KEYSTONE_PLATFORM=${{ matrix.platform }} - export KEYSTONE_BITS=${{ matrix.bits }} - export QEMU_PORT=$(( RANDOM + 1024 )) - export LD_LIBRARY_PATH=build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/host/lib - screen -L -dmS qemu bash -c "make run 2>&1 | tee run.log" - - # TODO: check for connectivity instead of sleeping - sleep 60 - - export CALL_LOGFILE=cmd.log - echo "" > $CALL_LOGFILE - - KEYSTONE_COMMAND="modprobe keystone-driver" make call - KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call - KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call - KEYSTONE_COMMAND="poweroff" make call - - - name: Check expected - run: | - [[ -z $(diff cmd.log scripts/ci/expected.log) ]] - - - name: Upload run log - if: failure() - uses: actions/upload-artifact@v4 - with: - name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log - path: run.log - - - name: Upload cmd log - if: failure() - uses: actions/upload-artifact@v4 - with: - name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log - path: cmd.log - - test-mpfs: - runs-on: [self-hosted, mpfs] - environment: track - steps: - # We don't need submodules here since Keystone is a monorepo! - - name: Checkout Keystone - uses: actions/checkout@v4 - with: - submodules: 'false' - - - name: Restore build directory - uses: actions/download-artifact@v4 - with: - name: keystone-mpfs64-builddir - path: . - - - name: Decompress build directory - run: cat build.tar.xz | xz -d -T0 | tar -xf - - - # Test the firmware, first by flashing it - - name: Flash HSS + - name: Flash and check firmware env: - POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }} - POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }} - SC_INSTALL_DIR: ${{ vars.SC_INSTALL_DIR }} + KEYSTONE_PLATFORM: ${{ matrix.platform }} + KEYSTONE_BITS: ${{ matrix.bits }} + LOGFILE: fw-program.log run: | - $POWER_ON_CMD - export FPGENPROG=$(which fpgenprog) - make -C build-mpfs64/buildroot.build/build/hss-v2023.06 program 2>/dev/null >program.log - $POWER_OFF_CMD - - # Check if we succeeded - [[ ! -z $(cat program.log | grep "mpfsBootmodeProgrammer completed successfully") ]] + if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh ]]; then + scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh + fi - - name: Upload HSS program log - if: failure() + - name: Upload firmware programming log + if: ${{ matrix.platform != 'generic' && failure() }} uses: actions/upload-artifact@v4 with: - name: test-keystone-mpfs64-prog-hss.log - path: program.log + name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-fw-program.log + path: fw-program.log - # And then verifying that we can actually get to the command line - - name: Check HSS ok + - name: Flash and check OS env: - POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }} - POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }} - FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }} + KEYSTONE_PLATFORM: ${{ matrix.platform }} + KEYSTONE_BITS: ${{ matrix.bits }} + LOGFILE: os-program.log run: | - # Collect serial output - TTYDEV=$($FIND_TTY_CMD 0) - screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run-hss.log" - $POWER_ON_CMD ; sleep 30 ; $POWER_OFF_CMD - screen -XS mpfs-tty quit - - # At least the first hart should have started - [[ ! -z $(cat run-hss.log | sed -e 's/\x1b\[[0-9;]*m//g' | grep "u54 State Change: \[Running\]") ]] + if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-os.sh ]]; then + scripts/ci/plat/${{ matrix.platform }}/flash-os.sh + fi - - name: Upload HSS run log - if: failure() + - name: Upload OS programming log + if: ${{ matrix.platform != 'generic' && failure() }} uses: actions/upload-artifact@v4 with: - name: test-keystone-mpfs64-run-hss.log - path: run-hss.log - - # Now we also need to flash the disk. First, get into usbdmsc - - name: Flash OS - env: - POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }} - POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }} - FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }} - run: | - # Wait for the board to come up a bit. We'll hammer it with serial - # input to ensure that we halt the boot at HSS - TTYDEV=$($FIND_TTY_CMD 0) - $POWER_ON_CMD - NOW=$(date +%s) - stty raw -echo 115200 < "$TTYDEV" - while [[ $(( $(date +%s) - $NOW )) -lt 10 ]]; do echo 'a' > "$TTYDEV" ; done - - echo "" > "$TTYDEV" - echo "usbdmsc" > "$TTYDEV" - - # Wait a bit for the USB to connect then flash - sleep 10 - FOUND_DEVICE="" - for d in /dev/sd? ; do - if [[ ! -z $(udevadm info --query=all -n "$d" | grep -i polarfire) ]]; then - FOUND_DEVICE="yes" - dd if=build-mpfs64/buildroot.build/images/sdcard.img of="$d" bs=4M oflag=direct - break - fi - done - - $POWER_OFF_CMD - [[ ! -z "$FOUND_DEVICE" ]] + name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-os-program.log + path: os-program.log - name: Test Keystone system env: - POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }} - POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }} - FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }} - KEYSTONE_IP: ${{ vars.BOARD_IP_MPFS }} + KEYSTONE_PLATFORM: ${{ matrix.platform }} + KEYSTONE_BITS: ${{ matrix.bits }} + LOGFILE: run.log + CMD_LOGFILE: cmd.log run: | - # Fix permissions on the key - chmod 600 build-mpfs64/buildroot.build/target/root/.ssh/id-rsa - - # Start the board - TTYDEV=$($FIND_TTY_CMD 1) - export KEYSTONE_PLATFORM=mpfs - export KEYSTONE_BITS=64 - screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run.log" - $POWER_ON_CMD - - # TODO: check for connectivity instead of sleeping - sleep 60 - - export CALL_LOGFILE=cmd.log - echo "" > $CALL_LOGFILE - - KEYSTONE_COMMAND="modprobe keystone-driver" make call - KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call - # Todo: attestation does not yet work in mpfs - #KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call - - $POWER_OFF_CMD - screen -XS mpfs-tty quit + scripts/ci/plat/${{ matrix.platform }}/test.sh - name: Check expected run: | - [[ -z $(diff cmd.log scripts/ci/expected-mpfs.log) ]] + [[ -z $(diff -wB cmd.log scripts/ci/plat/${{ matrix.platform }}/expected.log) ]] - name: Upload run log if: failure() uses: actions/upload-artifact@v4 with: - name: test-keystone-mpfs64-run.log + name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log path: run.log - name: Upload cmd log if: failure() uses: actions/upload-artifact@v4 with: - name: test-keystone-mpfs64-cmd.log + name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log path: cmd.log diff --git a/.gitmodules b/.gitmodules index 88a168640..b2c08c24f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "overlays/microchip"] path = overlays/microchip url = https://github.com/linux4microchip/buildroot-external-microchip +[submodule "scripts/ci/utils/relay_ft245r"] + path = scripts/ci/utils/relay_ft245r + url = https://github.com/vpatron/relay_ft245r diff --git a/mkutils/plat/cva6/run.mk b/mkutils/plat/cva6/run.mk index 28838fc15..cae910f80 100644 --- a/mkutils/plat/cva6/run.mk +++ b/mkutils/plat/cva6/run.mk @@ -5,10 +5,9 @@ PAYLOAD = $(BUILDROOT_BUILDDIR)/images/fw_payload.bin -SDDEVICE_PART1 = $(SD_DEVICE)1 -SDDEVICE_PART2 = $(SD_DEVICE)2 - - +KERNEL = $(BUILDROOT_BUILDDIR)/images/uImage +SDDEVICE_PART1 = $(shell lsblk $(SD_DEVICE) -no PATH | head -2 | tail -1) +SDDEVICE_PART2 = $(shell lsblk $(SD_DEVICE) -no PATH | head -3 | tail -1) flash: $(SD_DEVICE) $(info PAYLOAD INFORMATION) @@ -16,5 +15,20 @@ flash: $(SD_DEVICE) $(info $(SD_DEVICE)) $(info $(SDDEVICE_PART1)) $(info $(SDDEVICE_PART2)) - sgdisk --clear -g --new=1:2048:40M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE) + sgdisk --clear -g --new=1:2048:4M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE) dd if=$(PAYLOAD) of=$(SDDEVICE_PART1) status=progress oflag=sync bs=1M + dd if=$(KERNEL) of=$(SDDEVICE_PART2) status=progress oflag=sync bs=1M + +CALL_LOGFILE ?= $(shell mktemp) +call: + $(call log,info,Calling command on the CVA6 board) + ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \ + grep -v "Warning: Permanently added" | tee -a $(CALL_LOGFILE) + +debug-connect: + $(call log,info,Connecting to OpenOCD) + $(BUILDROOT_BUILDDIR)/host/bin/riscv64-buildroot-linux-gnu-gdb \ + -iex "set KEYSTONE=$(KEYSTONE)" \ + -x $(KEYSTONE)/scripts/gdb/cva6.cfg diff --git a/mkutils/plat/mpfs/run.mk b/mkutils/plat/mpfs/run.mk index 8b81a0ad7..e2162950d 100644 --- a/mkutils/plat/mpfs/run.mk +++ b/mkutils/plat/mpfs/run.mk @@ -20,7 +20,7 @@ run: CALL_LOGFILE ?= $(shell mktemp) call: - $(call log,info,Calling command in QEMU) + $(call log,info,Calling command on the MPFS board) ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \ diff --git a/overlays/keystone/board/cva6/configs/linux64-cva6-defconfig b/overlays/keystone/board/cva6/configs/linux64-cva6-defconfig index 91c48b7f9..b3124763e 100644 --- a/overlays/keystone/board/cva6/configs/linux64-cva6-defconfig +++ b/overlays/keystone/board/cva6/configs/linux64-cva6-defconfig @@ -2,15 +2,16 @@ CONFIG_DEFAULT_HOSTNAME="ariane-fpga" # CONFIG_CROSS_MEMORY_ATTACH is not set CONFIG_NAMESPACES=y CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="${BUILD_ROOT}/buildroot.build/images/rootfs.cpio" +CONFIG_INITRAMFS_SOURCE="${BR_BINARIES_DIR}/rootfs.cpio" CONFIG_EMBEDDED=y CONFIG_SMP=y CONFIG_HZ_100=y CONFIG_CMDLINE="earlyprintk initcall_debug" +# CONFIG_GCC_PLUGINS is not set CONFIG_MODULES=y -# CONFIG_BLK_DEV_BSG is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_COMPACTION is not set +CONFIG_CMA=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -26,16 +27,13 @@ CONFIG_NETDEVICES=y CONFIG_VXLAN=y CONFIG_LOWRISC_DIGILENT_100MHZ=y CONFIG_XILINX_EMACLITE=y -CONFIG_MDIO_BCM_UNIMAC=y +CONFIG_REALTEK_PHY=y CONFIG_MDIO_BITBANG=y +CONFIG_MDIO_BCM_UNIMAC=y +CONFIG_MDIO_GPIO=y CONFIG_MDIO_BUS_MUX_GPIO=y -CONFIG_MDIO_BUS_MUX_MMIOREG=y CONFIG_MDIO_BUS_MUX_MULTIPLEXER=y -CONFIG_MDIO_GPIO=y -CONFIG_REALTEK_PHY=y -CONFIG_DEBUG=y -CONFIG_DEBUG_INFO=y -CONFIG_GDB_SCRIPTS=y +CONFIG_MDIO_BUS_MUX_MMIOREG=y # CONFIG_WLAN is not set # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set @@ -44,15 +42,13 @@ CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_UARTLITE=y -CONFIG_HVC_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y # CONFIG_HW_RANDOM is not set CONFIG_I2C=y CONFIG_I2C_OCORES=y CONFIG_SPI=y -CONFIG_SPI_SPIDEV=y -CONFIG_SPI_MASTER=y CONFIG_SPI_XILINX=y +CONFIG_SPI_SPIDEV=y CONFIG_GPIOLIB=y CONFIG_GPIO_SYSFS=y CONFIG_GPIO_XILINX=y @@ -61,9 +57,6 @@ CONFIG_POWER_RESET_GPIO_RESTART=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y CONFIG_USB_EHCI_HCD=y -CONFIG_USB_SUPPORT=y -CONFIG_USB_COMMON=y -CONFIG_USB_CONFIGFS=m CONFIG_MMC=y CONFIG_MMC_SPI=y CONFIG_NEW_LEDS=y @@ -87,7 +80,7 @@ CONFIG_NFS_V4_1=y CONFIG_NFS_V4_2=y CONFIG_CRYPTO_ECHAINIV=y # CONFIG_CRYPTO_HW is not set +CONFIG_DMA_CMA=y CONFIG_PRINTK_TIME=y CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_SECTION_MISMATCH=y -CONFIG_STACKTRACE=y \ No newline at end of file diff --git a/overlays/keystone/board/cva6/configs/uboot-cva6.config b/overlays/keystone/board/cva6/configs/uboot-cva6.config new file mode 100644 index 000000000..f6272b4c6 --- /dev/null +++ b/overlays/keystone/board/cva6/configs/uboot-cva6.config @@ -0,0 +1,4 @@ +CONFIG_MMC_WRITE=y +CONFIG_BOOTCOMMAND="mmc info; mmc read 90000000 100000 10000; setenv fdt_high 0xffffffffffffffff; bootm 90000000 - $(fdtcontroladdr)" +CONFIG_USE_BOOTARGS=y +CONFIG_BOOTARGS="cma=128M" diff --git a/overlays/keystone/board/cva6/post-image.sh b/overlays/keystone/board/cva6/post-image.sh new file mode 100755 index 000000000..edef2f2f6 --- /dev/null +++ b/overlays/keystone/board/cva6/post-image.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "usage: post-image.sh " + exit 1 +fi + +if [ $2 -eq 32 ]; then + UIMAGE_ADDRESS=0x80400000 +elif [ $2 -eq 64 ]; then + UIMAGE_ADDRESS=0x80200000 +else + echo "invalid xlen" + exit 1 +fi + +# Generate uboot image +gzip -9 -k --force $1/Image > $1/Image.gz +$BUILDROOT_BUILDDIR/host/bin/mkimage -A riscv -O linux -T kernel -a $UIMAGE_ADDRESS -e $UIMAGE_ADDRESS -C gzip -n "CV$2A6Linux" -d $1/Image.gz $1/uImage diff --git a/overlays/keystone/configs/riscv64_cva6_defconfig b/overlays/keystone/configs/riscv64_cva6_defconfig index 44758e4ea..e265be169 100644 --- a/overlays/keystone/configs/riscv64_cva6_defconfig +++ b/overlays/keystone/configs/riscv64_cva6_defconfig @@ -12,6 +12,8 @@ BR2_TARGET_GENERIC_ROOT_PASSWD="sifive" BR2_SYSTEM_BIN_SH_BASH=y BR2_SYSTEM_DHCP="eth0" BR2_ROOTFS_POST_BUILD_SCRIPT="$(BR2_EXTERNAL_KEYSTONE_PATH)/board/cva6/post-build.sh" +BR2_ROOTFS_POST_IMAGE_SCRIPT="$(BR2_EXTERNAL_KEYSTONE_PATH)/board/cva6/post-image.sh" +BR2_ROOTFS_POST_SCRIPT_ARGS="64" BR2_LINUX_KERNEL=y BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_KEYSTONE_PATH)/board/cva6/configs/linux64-cva6-defconfig" @@ -25,7 +27,15 @@ BR2_TARGET_OPENSBI_CUSTOM_VERSION_VALUE="1.1" BR2_TARGET_OPENSBI_PLAT="fpga/ariane" # BR2_TARGET_OPENSBI_INSTALL_DYNAMIC_IMG is not set # BR2_TARGET_OPENSBI_INSTALL_JUMP_IMG is not set -BR2_TARGET_OPENSBI_LINUX_PAYLOAD=y +BR2_TARGET_OPENSBI_UBOOT_PAYLOAD=y +BR2_TARGET_UBOOT=y +BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y +BR2_TARGET_UBOOT_CUSTOM_GIT=y +BR2_TARGET_UBOOT_CUSTOM_REPO_URL="https://github.com/openhwgroup/u-boot" +BR2_TARGET_UBOOT_CUSTOM_REPO_VERSION="cva6" +BR2_TARGET_UBOOT_BOARD_DEFCONFIG="openhwgroup_cv64a6_genesysII" +BR2_TARGET_UBOOT_CONFIG_FRAGMENT_FILES="$(BR2_EXTERNAL_KEYSTONE_PATH)/board/cva6/configs/uboot-cva6.config" +BR2_PACKAGE_HOST_UBOOT_TOOLS=y BR2_TARGET_KEYSTONE_SM=y BR2_PACKAGE_KEYSTONE_DRIVER=y BR2_PACKAGE_HOST_KEYSTONE_SDK=y diff --git a/scripts/ci/configs/global.sh b/scripts/ci/configs/global.sh new file mode 100644 index 000000000..eb253ac8a --- /dev/null +++ b/scripts/ci/configs/global.sh @@ -0,0 +1,13 @@ + +####################################################### +## Globally known parameters (not machine dependent) ## +####################################################### + +# mpfs +export TTY_IDVENDOR_mpfs="10c4" +export TTY_IDPRODUCT_mpfs="ea71" + +# cva6 +export TTY_IDVENDOR_cva6="0403" +export TTY_IDPRODUCT_cva6="6001" + diff --git a/scripts/ci/configs/track.sh b/scripts/ci/configs/track.sh new file mode 100644 index 000000000..be4193f5d --- /dev/null +++ b/scripts/ci/configs/track.sh @@ -0,0 +1,22 @@ + +# Global configuration +export RELAY_SERIAL="AH02O23H" + +export RELAY_ID_global=1 +export RELAY_ID_mpfs=8 +export RELAY_ID_cva6=4 + +# MPFS configuration + +export SC_INSTALL_DIR="/opt/microchip/SoftConsole-v2022.2-RISC-V-747/" +export FPGENPROG=$(which fpgenprog) + +export HOST_IP_mpfs="10.42.0.1" +export BOARD_IP_mpfs="10.42.0.205" + +# CVA6 configuration + +export TFTP_DIR="/srv/tftp" + +export HOST_IP_cva6="10.42.1.1" +export BOARD_IP_cva6="10.42.1.171" diff --git a/scripts/ci/expected-mpfs.log b/scripts/ci/plat/cva6/expected.log similarity index 99% rename from scripts/ci/expected-mpfs.log rename to scripts/ci/plat/cva6/expected.log index a9ef4c0d4..6cb4b2318 100644 --- a/scripts/ci/expected-mpfs.log +++ b/scripts/ci/plat/cva6/expected.log @@ -1,4 +1,3 @@ - Verifying archive integrity... MD5 checksums are OK. All good. Uncompressing Keystone Enclave Package testing test-stack diff --git a/scripts/ci/plat/cva6/flash-firmware.sh b/scripts/ci/plat/cva6/flash-firmware.sh new file mode 100755 index 000000000..887af15bb --- /dev/null +++ b/scripts/ci/plat/cva6/flash-firmware.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -e + +# Source global test configuration file +. scripts/ci/test-setup.sh + +if [[ -z "$KEYSTONE_BITS" ]]; then + echo "KEYSTONE_BITS undefined" + exit 1 +fi + +FIRMWARE_FILENAME="build-cva6$KEYSTONE_BITS/buildroot.build/images/fw_payload.bin" +get_platform_var HOST_IP +get_platform_var BOARD_IP + +########### +## Flash ## +########### +set -x + +## ROM phase +TTYDEV=$(find_tty 0) +touch "$LOGFILE" +start_record_tty "$TTYDEV" 115200 "$LOGFILE" cva6-tty + +# Wait for bootrom break +power_on +wait_for "Hit any key to enter update mode" +echo -n 'a' > "$TTYDEV" +sleep 1 + +# Send size +FILESIZE=$(stat --printf="%s" "$FIRMWARE_FILENAME") +printf "0: %.8x" "$FILESIZE" | sed -E 's/0: (..)(..)(..)(..)/0: \4\3\2\1/' | xxd -r -g0 > "$TTYDEV" +sleep 1 + +# Send bytes +dd if="$FIRMWARE_FILENAME" of="$TTYDEV" + +## Uboot phase (TFTP) + +# Wait for uboot prompt and send firmware image +wait_for "Hit any key to stop autoboot" +echo 'a' > "$TTYDEV" + +rm -f "$TFTP_DIR/fw_payload.bin" +cp "$FIRMWARE_FILENAME" "$TFTP_DIR/fw_payload.bin" + +echo "setenv serverip $HOST_IP" > "$TTYDEV" ; sleep 1 +echo "setenv ipaddr $BOARD_IP" > "$TTYDEV" ; sleep 1 + +echo "tftp fw_payload.bin" > "$TTYDEV" +wait_for "=>" +echo "mmc write 80200000 800 1800" > "$TTYDEV" +wait_for "=>" + +stop_record_tty cva6-tty +power_off +exit 0 diff --git a/scripts/ci/plat/cva6/flash-os.sh b/scripts/ci/plat/cva6/flash-os.sh new file mode 100755 index 000000000..0a43d0b46 --- /dev/null +++ b/scripts/ci/plat/cva6/flash-os.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +# Source global test configuration file +. scripts/ci/test-setup.sh + +if [[ -z "$KEYSTONE_BITS" ]]; then + echo "KEYSTONE_BITS undefined" + exit 1 +fi + +OS_FILENAME="build-cva6$KEYSTONE_BITS/buildroot.build/images/uImage" +get_platform_var HOST_IP +get_platform_var BOARD_IP + +########### +## Flash ## +########### +set -x + +TTYDEV=$(find_tty 0) +touch "$LOGFILE" +start_record_tty "$TTYDEV" 115200 "$LOGFILE" cva6-tty + +power_on +wait_for "Hit any key to stop autoboot" +echo 'a' > "$TTYDEV" + +rm -f "$TFTP_DIR/uImage" +cp "$OS_FILENAME" "$TFTP_DIR/uImage" + +# Configure TFTP +echo "setenv serverip $HOST_IP" > "$TTYDEV" ; sleep 1 +echo "setenv ipaddr $BOARD_IP" > "$TTYDEV" ; sleep 1 + +echo "tftp uImage" > "$TTYDEV" +wait_for "=>" +echo "mmc write 80200000 100000 10000" > "$TTYDEV" +wait_for "=>" + +stop_record_tty cva6-tty +power_off +exit 0 diff --git a/scripts/ci/plat/cva6/test.sh b/scripts/ci/plat/cva6/test.sh new file mode 100755 index 000000000..4509a793a --- /dev/null +++ b/scripts/ci/plat/cva6/test.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e + +# Source global test configuration file +. scripts/ci/test-setup.sh + +if [[ -z "$CMD_LOGFILE" ]]; then + echo "CMD_LOGFILE undefined" + exit 1 +fi + +if [[ -z "$KEYSTONE_BITS" ]]; then + echo "KEYSTONE_BITS undefined" + exit 1 +fi + +get_platform_var BOARD_IP + +############### +## Run tests ## +############### +set -x + +# Fix permissions on the key +chmod 600 "build-cva6$KEYSTONE_BITS/buildroot.build/target/root/.ssh/id-rsa" + +# Start the board +export KEYSTONE_PLATFORM=cva6 +export KEYSTONE_IP="$BOARD_IP" + +TTYDEV=$(find_tty 0) +start_record_tty "$TTYDEV" 115200 "$LOGFILE" cva6-tty +power_on + +# TODO: check for connectivity instead of sleeping +sleep 300 + +export CALL_LOGFILE="$CMD_LOGFILE" +touch "$CALL_LOGFILE" + +KEYSTONE_COMMAND="modprobe keystone-driver" make call +KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call +# TODO: attestation does not yet work in cva6 +#KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call + +power_off +stop_record_tty cva6-tty +exit 0 diff --git a/scripts/ci/expected.log b/scripts/ci/plat/generic/expected.log similarity index 99% rename from scripts/ci/expected.log rename to scripts/ci/plat/generic/expected.log index a4bcaf3b6..35e3053d8 100644 --- a/scripts/ci/expected.log +++ b/scripts/ci/plat/generic/expected.log @@ -1,4 +1,3 @@ - Verifying archive integrity... MD5 checksums are OK. All good. Uncompressing Keystone Enclave Package testing test-stack diff --git a/scripts/ci/plat/generic/test.sh b/scripts/ci/plat/generic/test.sh new file mode 100755 index 000000000..c9773e5a0 --- /dev/null +++ b/scripts/ci/plat/generic/test.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +if [[ -z "$CMD_LOGFILE" ]]; then + echo "CMD_LOGFILE undefined" + exit 1 +fi + +if [[ -z "$KEYSTONE_BITS" ]]; then + echo "KEYSTONE_BITS undefined" + exit 1 +fi + +############### +## Run tests ## +############### +set -x + +# Fix permissions on the key +chmod 600 "build-generic$KEYSTONE_BITS/buildroot.build/target/root/.ssh/id-rsa" + +# Launch QEMU +export KEYSTONE_PLATFORM="generic" +export QEMU_PORT=$(( RANDOM + 1024 )) +export LD_LIBRARY_PATH="build-generic$KEYSTONE_BITS/buildroot.build/host/lib" +screen -L -dmS qemu bash -c "make run 2>&1 | tee $LOGFILE" + +# TODO: check for connectivity instead of sleeping +sleep 60 + +export CALL_LOGFILE="$CMD_LOGFILE" +echo "" > "$CALL_LOGFILE" + +KEYSTONE_COMMAND="modprobe keystone-driver" make call +KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call +KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call +KEYSTONE_COMMAND="poweroff" make call + +screen -S qemu -X quit +exit 0 diff --git a/scripts/ci/plat/mpfs/expected.log b/scripts/ci/plat/mpfs/expected.log new file mode 100644 index 000000000..6cb4b2318 --- /dev/null +++ b/scripts/ci/plat/mpfs/expected.log @@ -0,0 +1,17 @@ +Verifying archive integrity... MD5 checksums are OK. All good. +Uncompressing Keystone Enclave Package +testing test-stack +testing test-loop +testing test-malloc +testing test-long-nop +testing test-fibonacci +testing test-fib-bench +testing test-attestation +Attestation report is invalid +testing test-untrusted +Enclave said: hello world! +Enclave said: 2nd hello world! +Enclave said value: 13 +Enclave said value: 20 +testing test-data-sealing +Enclave said: Sealing key derivation successful! diff --git a/scripts/ci/plat/mpfs/flash-firmware.sh b/scripts/ci/plat/mpfs/flash-firmware.sh new file mode 100755 index 000000000..28b34863e --- /dev/null +++ b/scripts/ci/plat/mpfs/flash-firmware.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +# Source global test configuration file +. scripts/ci/test-setup.sh + +########### +## Flash ## +########### +set -x + +power_on +make -C build-mpfs64/buildroot.build/build/hss-v2023.06 program 2>/dev/null > "$LOGFILE" +power_off + +# Check if flashing was successful +[[ ! -z $(cat "$LOGFILE" | grep "mpfsBootmodeProgrammer completed successfully") ]] + +########### +## Check ## +########### + +TTYDEV=$(find_tty 0) +start_record_tty "$TTYDEV" 115200 "$LOGFILE" mpfs-tty +power_on ; sleep 30; power_off +stop_record_tty mpfs-tty + +# At least the first hart should have started +[[ ! -z $(cat "$LOGFILE" | sed -e 's/\x1b\[[0-9;]*m//g' | grep "u54 State Change: \[Running\]") ]] + +exit 0 diff --git a/scripts/ci/plat/mpfs/flash-os.sh b/scripts/ci/plat/mpfs/flash-os.sh new file mode 100755 index 000000000..b34eeebeb --- /dev/null +++ b/scripts/ci/plat/mpfs/flash-os.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +# Source global test configuration file +. scripts/ci/test-setup.sh + +########### +## Flash ## +########### +set -x + +# Wait for the board to come up a bit. We'll hammer it with serial +# input to ensure that we halt the boot at HSS + +TTYDEV=$(find_tty 0) +configure_tty "$TTYDEV" 115200 + +power_on +NOW=$(date +%s) + +# Disable output when actually hammering cause this is spammy +set +x +while [[ $(( $(date +%s) - $NOW )) -lt 10 ]]; do echo 'a' > "$TTYDEV" ; done +set -x + +# Board should have halted, kick it into flash update mode + +echo "" > "$TTYDEV" +echo "usbdmsc" > "$TTYDEV" + +# Wait a bit for the USB to connect then flash +sleep 10 +FOUND_DEVICE="" +for d in /dev/sd? ; do + if [[ ! -z $(udevadm info --query=all -n "$d" | grep -i polarfire) ]]; then + FOUND_DEVICE="yes" + dd if="build-mpfs64/buildroot.build/images/sdcard.img" of="$d" bs=4M oflag=direct + break + fi +done + +power_off +[[ ! -z "$FOUND_DEVICE" ]] + +exit 0 diff --git a/scripts/ci/plat/mpfs/test.sh b/scripts/ci/plat/mpfs/test.sh new file mode 100755 index 000000000..46ce55ee7 --- /dev/null +++ b/scripts/ci/plat/mpfs/test.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -e + +# Source global test configuration file +. scripts/ci/test-setup.sh + +if [[ -z "$CMD_LOGFILE" ]]; then + echo "CMD_LOGFILE undefined" + exit 1 +fi + +get_platform_var BOARD_IP + +############### +## Run tests ## +############### +set -x + +# Fix permissions on the key +chmod 600 build-mpfs64/buildroot.build/target/root/.ssh/id-rsa + +# Start the board +export KEYSTONE_PLATFORM=mpfs +export KEYSTONE_BITS=64 +export KEYSTONE_IP="$BOARD_IP" + +TTYDEV=$(find_tty 1) +start_record_tty "$TTYDEV" 115200 "$LOGFILE" mpfs-tty +power_on + +# TODO: check for connectivity instead of sleeping +sleep 60 + +export CALL_LOGFILE="$CMD_LOGFILE" +touch "$CALL_LOGFILE" + +KEYSTONE_COMMAND="modprobe keystone-driver" make call +KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call +# TODO: attestation does not yet work in mpfs +#KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call + +power_off +stop_record_tty mpfs-tty +exit 0 diff --git a/scripts/ci/test-setup.sh b/scripts/ci/test-setup.sh new file mode 100755 index 000000000..f2d19dfa3 --- /dev/null +++ b/scripts/ci/test-setup.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -e + +# Generic setup script sourced by all CI test scripts. This allows us to have +# convenient global configuration options which are always available in +# consumer scripts. + +################### +## Configuration ## +################### + +# Check if we were called from the right location +if [[ $(basename "$PWD") != "keystone" ]]; then + echo "CI scripts must be called from base Keystone directory" + exit 1 +fi + +# Check if we have the right environment +if [[ -z "$KEYSTONE_PLATFORM" ]]; then + echo "KEYSTONE_PLATFORM undefined" + exit 1 +fi + +if [[ -z "$LOGFILE" ]]; then + echo "LOGFILE undefined" + exit 1 +fi + +# Source global configuration +. scripts/ci/configs/global.sh + +# Source runner-specific configuration +if [[ ! -f "scripts/ci/configs/$(hostname).sh" ]]; then + echo "No configuration file for this machine" + exit 1 +fi +. "scripts/ci/configs/$(hostname).sh" + + +###################### +## Useful functions ## +###################### + +# Utility for expanding variables of the form VAR_, where plat is one of +# the supported Keytone platforms. Lots of variables like these live in the CI +# configuration files. + +function get_platform_var { + if [[ "$#" -ne 1 ]]; then + echo "usage: get_platform_var [varname]" + exit 1 + fi + + _VARNAME="$1_$KEYSTONE_PLATFORM" + export "$1"="${!_VARNAME}" + unset _VARNAME + + if [[ -z "${!1}" ]]; then + echo "Variable $1 not defined for platform $KEYSTONE_PLATFORM" + exit 1 + fi +} + +# Power functions + +get_platform_var RELAY_ID + +function power_on { + ./scripts/ci/utils/relay_power.py "$RELAY_SERIAL" "$RELAY_ID" on +} + +function power_off { + ./scripts/ci/utils/relay_power.py "$RELAY_SERIAL" "$RELAY_ID" off +} + +# Serial functions + +get_platform_var TTY_IDVENDOR +get_platform_var TTY_IDPRODUCT + +function find_tty { + if [[ "$#" -ne 1 ]]; then + echo "usage: find_tty [index]" + exit 1 + fi + + echo $(./scripts/ci/utils/find_tty.sh "$TTY_IDVENDOR" "$TTY_IDPRODUCT" "$1") +} + +function configure_tty { + if [[ "$#" -ne 2 ]]; then + echo "usage: configure_tty [tty_file] [baud]" + exit 1 + fi + + if [[ ! -c "$1" ]]; then + echo "$1 is not a valid character device" + exit 1 + fi + + stty raw -echo "$2" < "$1" +} +export -f configure_tty + +function start_record_tty { + if [[ "$#" -ne 4 ]]; then + echo "usage: start_record_tty [tty_file] [baud] [output_filename] [id]" + exit 1 + fi + + screen -L -dmS "$4" bash -c "configure_tty $1 $2 ; cat $1 > $3" +} + +function stop_record_tty { + if [[ "$#" -ne 1 ]]; then + echo "usage: stop_record_tty [id]" + exit 1 + fi + + screen -XS "$1" quit + screen -wipe || true +} + +function wait_for { + if [[ "$#" -ne 1 ]]; then + echo "usage: wait_for [pattern]" + exit 1 + fi + + ./scripts/ci/utils/wait_for.py "$LOGFILE" "$1" +} + +############# +## Cleanup ## +############# + +# Make sure we turn off the boards if we die early +trap power_off EXIT diff --git a/scripts/ci/utils/find_tty.sh b/scripts/ci/utils/find_tty.sh new file mode 100755 index 000000000..19d121883 --- /dev/null +++ b/scripts/ci/utils/find_tty.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +if [[ "$#" -ne 3 ]]; then + echo "usage: find_tty.sh [idvendor] [idproduct] [index]" + exit 1 +fi + +TTY_IDVENDOR="$1"; shift +TTY_IDPRODUCT="$1"; shift + +# See if we can find the correct TTYs +TTYS=() + +for f in /sys/class/tty/ttyUSB* ; do + if [[ $(cat $f/../../../../idVendor) == "$TTY_IDVENDOR" ]] && \ + [[ $(cat $f/../../../../idProduct) == "$TTY_IDPRODUCT" ]]; then + # This is one of the TTYs we are looking for + TTYS+=("$(basename $f)") + fi +done + +if [[ "$1" -lt "${#TTYS[@]}" ]]; then + echo "/dev/${TTYS[$1]}" + exit 0 +fi + +exit 1 + diff --git a/scripts/ci/utils/relay_ft245r b/scripts/ci/utils/relay_ft245r new file mode 160000 index 000000000..046854b2a --- /dev/null +++ b/scripts/ci/utils/relay_ft245r @@ -0,0 +1 @@ +Subproject commit 046854b2ac02c69f1037ec24cd17b3700b1e3906 diff --git a/scripts/ci/utils/relay_power.py b/scripts/ci/utils/relay_power.py new file mode 100755 index 000000000..49836c816 --- /dev/null +++ b/scripts/ci/utils/relay_power.py @@ -0,0 +1,58 @@ +#!/bin/python3 + +from relay_ft245r import relay_ft245r +import sys +import time + +# Check arguments +if len(sys.argv) != 4: + print('Usage: relay_power.py [serial#] [port] [on/off]') + sys.exit() + +serial_number = sys.argv[1] +port = int(sys.argv[2]) +state = sys.argv[3].strip().lower() + +if state not in ['on', 'off', 'reboot']: + print('Unrecognized command ', state) + sys.exit() + +rb = relay_ft245r.FT245R() +dev_list = rb.list_dev() + +# list of FT245R devices are returned +if len(dev_list) == 0: + print('No FT245R devices found') + sys.exit() + +found = False +for dev in dev_list: + # This is the one we should use + if dev.serial_number == serial_number: + found = True + break + +if found: + rb.connect(dev) + if state == 'on': + if not rb.getstatus(port): + rb.switchoff(port) + + rb.switchon(port) + else: + # Cursed: for some reason, we need to switch the port on which + # is effectively a no-op before we can turn it off. We don't + # want to do this spuriously though (fast switches may be bad + # in case the port is already off) so we check this case. + if rb.getstatus(port): + rb.switchon(port) + + rb.switchoff(port) + + if state == 'reboot': + time.sleep(0.1) + rb.switchon(port) + +else: + print('FT245R with specified serial not found: ', SERIAL_NUMBER) + sys.exit() diff --git a/scripts/ci/utils/wait_for.py b/scripts/ci/utils/wait_for.py new file mode 100755 index 000000000..85a5fbbbc --- /dev/null +++ b/scripts/ci/utils/wait_for.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 -u + +import sys +import os + +if len(sys.argv) != 3: + print('usage: wait_for.py [file] [pattern]') + exit(1) + +file = open(sys.argv[1], 'rb', buffering=0) +file.seek(0, os.SEEK_END) +pattern = sys.argv[2].encode('utf-8') + +index = 0 +while True: + if index == len(pattern): + exit(0) + + n = os.read(file.fileno(), 1) + if n is None: + exit(1) + + if len(n) > 0: + if n[0] == pattern[index]: + index += 1 + else: + index = 0 diff --git a/sm/src/platform/fpga/ariane/platform.h b/sm/src/platform/fpga/ariane/platform.h index 09fa7841f..5f3edb595 100644 --- a/sm/src/platform/fpga/ariane/platform.h +++ b/sm/src/platform/fpga/ariane/platform.h @@ -13,7 +13,7 @@ struct platform_enclave_data{ // SM configuration #define SMM_BASE 0x80000000 -#define SMM_SIZE 0x200000 +#define SMM_SIZE 0x100000 // PMP configuration #define PMP_N_REG 8