From 3918e518600d40ced04ce5a2ad619675f6f3d656 Mon Sep 17 00:00:00 2001 From: Jonas Dann Date: Tue, 12 Nov 2024 13:27:43 +0100 Subject: [PATCH 1/3] Fixed flow_alveo.sh environment issue --- util/flow_alveo.sh | 4 +--- util/insmod_local.sh | 10 ++++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 util/insmod_local.sh diff --git a/util/flow_alveo.sh b/util/flow_alveo.sh index 9066f064..c888e8a4 100755 --- a/util/flow_alveo.sh +++ b/util/flow_alveo.sh @@ -101,10 +101,8 @@ if [ $DRV_INSERT -eq 1 ]; then echo "*** Loading the driver ..." echo " ** " - qsfp_ip="DEVICE_1_IP_ADDRESS_HEX_$QSFP_PORT" - qsfp_mac="DEVICE_1_MAC_ADDRESS_$QSFP_PORT" - parallel-ssh -H "$hostlist" -x '-tt' "sudo insmod $BASE_PATH/../$DRV_PATH/coyote_drv.ko ip_addr=\$$qsfp_ip mac_addr=\$$qsfp_mac" + parallel-ssh -H "$hostlist" -x '-tt' "cd $BASE_PATH/../driver && ../util/insmod_local.sh" echo "*** Driver loaded" echo " ** " diff --git a/util/insmod_local.sh b/util/insmod_local.sh new file mode 100644 index 00000000..613f846a --- /dev/null +++ b/util/insmod_local.sh @@ -0,0 +1,10 @@ +CLI_PATH=/opt/sgrt/cli + +IP_address=$($CLI_PATH/sgutil get network -d 1 | awk '$1 == "1:" {print $2}') +MAC_address=$($CLI_PATH/sgutil get network -d 1 | awk '$1 == "1:" {print $3}' | tr -d '()') +qsfp_ip=$($CLI_PATH/common/address_to_hex IP $IP_address) +qsfp_mac=$($CLI_PATH/common/address_to_hex MAC $MAC_address) +echo "** IP_ADDRESS: $qsfp_ip" +echo "** MAC_ADDRESS: $qsfp_mac" + +sudo insmod coyote_drv.ko ip_addr=$qsfp_ip mac_addr=$qsfp_mac \ No newline at end of file From 0d69127e8df76e5b2fd784b746bd2a102aa70183 Mon Sep 17 00:00:00 2001 From: Jonas Dann Date: Tue, 12 Nov 2024 15:29:25 +0100 Subject: [PATCH 2/3] Renamed program scripts and added some documentation and argument checks for stability --- README.md | 8 +++-- program_coyote.sh => program_hacc_local.sh | 0 util/flow_alveo.sh => program_hacc_remote.sh | 38 ++++++++++++-------- util/insmod_local.sh | 2 ++ 4 files changed, 30 insertions(+), 18 deletions(-) rename program_coyote.sh => program_hacc_local.sh (100%) rename util/flow_alveo.sh => program_hacc_remote.sh (78%) diff --git a/README.md b/README.md index 97fe4d0e..24110914 100644 --- a/README.md +++ b/README.md @@ -168,12 +168,12 @@ $ make bitgen in the original build-directory as described before. ## Deploying on the ETHZ HACC-cluster -The ETHZ HACC is a premiere cluster for research in systems, architecture and applications (https://github.com/fpgasystems/hacc/tree/main). Its hardware equipment provides the ideal environment to run Coyote-based experiments, since users can book up to 10 servers with U55C-accelerator cards connected via a fully switched 100G-network. User accounts for this platform can be obtained following the explanation on the previously cited homepage. +The ETHZ HACC is a premiere cluster for research in systems, architecture, and applications (https://github.com/fpgasystems/hacc/tree/main). Its hardware equipment provides the ideal environment to run Coyote-based experiments, since users can book up to 10 servers with U55C-accelerator cards connected via a fully switched 100G-network. User accounts for this platform can be obtained following the explanation on the previously cited homepage. -The interaction with the HACC-cluster can be simplified by using the sgutil-run time commands. They also allow to easily program the accelerator with a Coyote-bitstreamd and insert the driver. For this purpose, the script `program_coyote.sh` has been generated. Under the assumption that the hardware-project has been created in `examples_hw/build` and the driver is already compiled in `driver`, the workflow should look like this: +The interaction with the HACC-cluster can be simplified by using the sgutil-run time commands. They also allow to easily program the accelerator with a Coyote-bitstreamd and insert the driver. For this purpose, the scripts `program_hacc_local.sh` and `program_hacc_remote.sh` have been created. Under the assumption that the hardware-project has been created in `examples_hw/build` and the driver is already compiled in `driver`, the workflow should look like this: ~~~ -$ bash program_coyote.sh examples_hw/build/bitstreams/cyt_top.bit driver/coyote_drv.ko +$ bash program_hacc_local.sh examples_hw/build/bitstreams/cyt_top.bit driver/coyote_drv.ko ~~~ Obviously, the paths to `cyt_top.bit` and `coyote_drv.ko` need to be adapted if a different build-structure has been chosen before. @@ -185,6 +185,8 @@ $ dmesg If the driver insertion went through, the last printed message should be `probe returning 0`. Furthermore, the dmesg-printout should contain a line `set network ip XXXXXXXX, mac YYYYYYYYYYYY`, which displays IP and MAC of the Coyote-NIC if networking has been enabled in the system configuration. +To program Coyote to a remote server, `program_hacc_remote.sh` may be used in the same way. Additionally, that script will ask for a list of server ids. + ## Publication #### If you use Coyote, cite us : diff --git a/program_coyote.sh b/program_hacc_local.sh similarity index 100% rename from program_coyote.sh rename to program_hacc_local.sh diff --git a/util/flow_alveo.sh b/program_hacc_remote.sh similarity index 78% rename from util/flow_alveo.sh rename to program_hacc_remote.sh index c888e8a4..36982193 100755 --- a/util/flow_alveo.sh +++ b/program_hacc_remote.sh @@ -4,23 +4,32 @@ ## Args ## -if [ "$1" == "-h" ]; then - echo "Usage: $0 " >&2 +if [ "$1" == "-h" ] || [ $# -eq 0 ]; then + echo "Usage: $0 [ []]" >&2 exit 0 fi -if ! [ -x "$(command -v vivado)" ]; then - echo "Vivado does NOT exist in the system." - exit 1 -fi - BASE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) PROGRAM_FPGA=1 DRV_INSERT=1 -BIT_PATH=$1 -DRV_PATH=$2 +BIT_PATH="${1%%.bit}" # Strip .bit +DRV_PATH=driver + +if [ ! -f ${BIT_PATH}.bit ]; then + echo "Bitstream ${BIT_PATH}.bit does not exist." + exit 1 +fi + +if ! [ -x "$(command -v vivado)" ]; then + echo "Vivado does NOT exist in the system." + exit 1 +fi + +if [ -n "$2" ]; then + DRV_PATH=$2 +fi if [ -z "$3" ]; then QSFP_PORT=0 @@ -32,7 +41,7 @@ fi ## Server IDs (u55c) ## -echo "*** Enter server IDs:" +echo "*** Enter U55C server IDs [1,10] (e.g., <3, 5>):" read -a SERVID BOARDSN=(XFL1QOQ1ATTYA XFL1O5FZSJEIA XFL1QGKZZ0HVA XFL11JYUKD4IA XFL1EN2C02C0A XFL1NMVTYXR4A XFL1WI3AMW4IA XFL1ELZXN2EGA XFL1W5OWZCXXA XFL1H2WA3T53A) @@ -53,7 +62,7 @@ alveo_program() BOARDSN=$3 DEVICENAME=$4 BITPATH=$5 - vivado -nolog -nojournal -mode batch -source program_alveo.tcl -tclargs $SERVERADDR $SERVERPORT $BOARDSN $DEVICENAME $BITPATH + vivado -nolog -nojournal -mode batch -source util/program_alveo.tcl -tclargs $SERVERADDR $SERVERPORT $BOARDSN $DEVICENAME $BITPATH } if [ $PROGRAM_FPGA -eq 1 ]; then @@ -72,7 +81,7 @@ if [ $PROGRAM_FPGA -eq 1 ]; then echo " ** " for servid in "${SERVID[@]}"; do boardidx=$(expr $servid - 1) - alveo_program alveo-u55c-$(printf "%02d" $servid) 3121 ${BOARDSN[boardidx]} xcu280_u55c_0 $BASE_PATH/../$BIT_PATH & + alveo_program alveo-u55c-$(printf "%02d" $servid) 3121 ${BOARDSN[boardidx]} xcu280_u55c_0 $BASE_PATH/$BIT_PATH & done wait @@ -97,12 +106,11 @@ if [ $DRV_INSERT -eq 1 ]; then echo "*** Compiling the driver ..." echo " ** " - parallel-ssh -H "$hostlist" "make -C $BASE_PATH/../$DRV_PATH" + parallel-ssh -H "$hostlist" "make -C $BASE_PATH/$DRV_PATH" echo "*** Loading the driver ..." echo " ** " - - parallel-ssh -H "$hostlist" -x '-tt' "cd $BASE_PATH/../driver && ../util/insmod_local.sh" + parallel-ssh -H "$hostlist" -x '-tt' "cd $BASE_PATH/$DRV_PATH && ../util/insmod_local.sh" echo "*** Driver loaded" echo " ** " diff --git a/util/insmod_local.sh b/util/insmod_local.sh index 613f846a..9bf4d530 100644 --- a/util/insmod_local.sh +++ b/util/insmod_local.sh @@ -1,3 +1,5 @@ +# Script to set up IP and MAC address environment variables and insert the driver because environment is not initialized when executing through ssh from a remote server (environment is only initialized when MOTD is shown). + CLI_PATH=/opt/sgrt/cli IP_address=$($CLI_PATH/sgutil get network -d 1 | awk '$1 == "1:" {print $2}') From ffaa745da2cd3f2805eb29e202aaf5895d4b554a Mon Sep 17 00:00:00 2001 From: Jonas Dann Date: Thu, 14 Nov 2024 16:04:36 +0100 Subject: [PATCH 3/3] Removed unnecessary script in util and moved program scripts to util --- README.md | 8 ++--- util/find.py | 33 ------------------- .../program_hacc_local.sh | 0 .../program_hacc_remote.sh | 2 +- 4 files changed, 5 insertions(+), 38 deletions(-) delete mode 100644 util/find.py rename program_hacc_local.sh => util/program_hacc_local.sh (100%) rename program_hacc_remote.sh => util/program_hacc_remote.sh (99%) diff --git a/README.md b/README.md index 24110914..8f18c48d 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,13 @@ in the original build-directory as described before. ## Deploying on the ETHZ HACC-cluster The ETHZ HACC is a premiere cluster for research in systems, architecture, and applications (https://github.com/fpgasystems/hacc/tree/main). Its hardware equipment provides the ideal environment to run Coyote-based experiments, since users can book up to 10 servers with U55C-accelerator cards connected via a fully switched 100G-network. User accounts for this platform can be obtained following the explanation on the previously cited homepage. -The interaction with the HACC-cluster can be simplified by using the sgutil-run time commands. They also allow to easily program the accelerator with a Coyote-bitstreamd and insert the driver. For this purpose, the scripts `program_hacc_local.sh` and `program_hacc_remote.sh` have been created. Under the assumption that the hardware-project has been created in `examples_hw/build` and the driver is already compiled in `driver`, the workflow should look like this: +The interaction with the HACC-cluster can be simplified by using the sgutil-run time commands. They also allow to easily program the accelerator with a Coyote-bitstreamd and insert the driver. For this purpose, the scripts `util/program_hacc_local.sh` and `util/program_hacc_remote.sh` have been created. Under the assumption that the hardware-project has been created in `examples_hw/build` and the driver is already compiled in `driver`, the workflow should look like this: ~~~ -$ bash program_hacc_local.sh examples_hw/build/bitstreams/cyt_top.bit driver/coyote_drv.ko +$ bash util/program_hacc_local.sh examples_hw/build/bitstreams/cyt_top.bit driver/coyote_drv.ko ~~~ -Obviously, the paths to `cyt_top.bit` and `coyote_drv.ko` need to be adapted if a different build-structure has been chosen before. +The paths to `cyt_top.bit` and `coyote_drv.ko` need to be adapted if a different build-structure has been chosen before. A successful completion of this process can be checked via a call to ~~~ @@ -185,7 +185,7 @@ $ dmesg If the driver insertion went through, the last printed message should be `probe returning 0`. Furthermore, the dmesg-printout should contain a line `set network ip XXXXXXXX, mac YYYYYYYYYYYY`, which displays IP and MAC of the Coyote-NIC if networking has been enabled in the system configuration. -To program Coyote to a remote server, `program_hacc_remote.sh` may be used in the same way. Additionally, that script will ask for a list of server ids. +To program Coyote to a remote server, `util/program_hacc_remote.sh` may be used in the same way. Additionally, that script will ask for a list of server ids (e.g., `3, 5`). ## Publication diff --git a/util/find.py b/util/find.py deleted file mode 100644 index 23192ad2..00000000 --- a/util/find.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import os, sys - -def getOutput(cmd): - return os.popen(cmd).read() - -if (len(sys.argv) <> 2): - print "usage: %s size_in_bytes" % sys.argv[0] -else: - maxSize = int(sys.argv[1]) - - revisions = getOutput("git rev-list HEAD").split() - - bigfiles = set() - for revision in revisions: - files = getOutput("git ls-tree -zrl %s" % revision).split('\0') - for file in files: - if file == "": - continue - splitdata = file.split() - commit = splitdata[2] - if splitdata[3] == "-": - continue - size = int(splitdata[3]) - path = splitdata[4] - if (size > maxSize): - bigfiles.add("%10d %s %s" % (size, commit, path)) - - bigfiles = sorted(bigfiles, reverse=True) - - for f in bigfiles: - print f diff --git a/program_hacc_local.sh b/util/program_hacc_local.sh similarity index 100% rename from program_hacc_local.sh rename to util/program_hacc_local.sh diff --git a/program_hacc_remote.sh b/util/program_hacc_remote.sh similarity index 99% rename from program_hacc_remote.sh rename to util/program_hacc_remote.sh index 36982193..09b80fdb 100755 --- a/program_hacc_remote.sh +++ b/util/program_hacc_remote.sh @@ -9,7 +9,7 @@ if [ "$1" == "-h" ] || [ $# -eq 0 ]; then exit 0 fi -BASE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BASE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/.. PROGRAM_FPGA=1 DRV_INSERT=1