![\"example](\"diagram.png\")
\n",
+ "\n",
+ "We will now demonstrate the use of HetSim for an example scenarios.
+ +sudo apt install g++-arm-linux-gnueabihfWe will build gem5 by running a convenience script inside scripts/.
%cd scripts
+!VERBOSE=0 CC=/usr/bin/gcc CXX=/usr/bin/g++ bash build-gem5.sh
+%cd ..
+Consider a programmable target composed of two types of PEs - worker and manager.
+ +
+We will make two changes to the existing model.
+%cd example/model
+# change the relevant define in params.h
+!sed -i 's/#define NUM_WORKER.*/#define NUM_WORKER 8/g' params.h
+!sed -i 's/#define WQ_DEPTH.*/#define WQ_DEPTH 6/g' params.h
+# generate Python bindings
+!make
+%cd ../..
+# this step should be done manually!
+!sed -i 's/\[1, 2, 3, 4\]/[1, 2, 3, 4, 5, 6, 7, 8]/g' spec/spec.json
+gem5 model and the emulation and TRE libraries%cd scripts
+!python2 populate_init_queues.py
+!VERBOSE=0 bash build-gem5.sh
+%cd ..
+Finally, we regenerate the compiler plugin and build the tracing library.
+ +%cd scripts
+!python generate_model.py ../spec/spec.json > /dev/null
+%cd ../tracer
+%mkdir -p build
+%cd build
+!cmake .. > /dev/null && make
+%cd ../..
+We will write a program to do vector addition on the target hardware.
+ +#include "params.h" // import parameters of target hardware as macros
+#include "util.h" // import primitive definitions
+#include <pthread.h>
+#include <sys/mman.h>
+void *work(void *arg) { // manager "spawns" worker threads with tid=1,2,3...
+ unsigned tid = *(unsigned *)(arg);
+ __register_core_id(tid);
+ ...
+}
+int main() {
+ __init_queues(WQ_DEPTH);
+ __register_core_id(0); // manager is assigned core-id 0
+ ...
+ __teardown_queues();
+ return 0;
+}
+The core_id must be the same across the application, spec.json and target.py.
# spec.json
+!grep -A1 -B1 "id" spec/spec.json
+# target.py
+!grep -A1 -B1 "id=" example/model/target.py
+main()¶Memory allocation is done at the beginning part of main().
// main memory allocation
+ // in this example, we are working with 3 float arrays each of size N
+ size_t RAM_SIZE_BYTES = 3 * N * sizeof(float);
+ char *ram = (char *)mmap((void *)(RAM_BASE_ADDR), RAM_SIZE_BYTES,
+ PROT_READ | PROT_WRITE | PROT_EXEC,
+ MAP_ANON | MAP_PRIVATE, 0, 0);
+// scratchpad memory allocation
+#ifdef EMULATION
+ // for emulation
+ char *dspm = (char *)mmap((void *)(SPM_BASE_ADDR), SPM_SIZE_BYTES,
+ PROT_READ | PROT_WRITE | PROT_EXEC,
+ MAP_ANON | MAP_PRIVATE, 0, 0);
+#else // !EMULATION
+ // the model uses physically-addressed scratchpad that does not need explicit allocation
+ char *dspm = (char *)SPM_BASE_ADDR;
+#endif // EMULATION
+We "allocate" the input and output vectors at the pre-allocated main memory and initialize them.
+// allocate the vectors and populate them
+ float *a = (float *)(ram);
+ float *b = (float *)(ram + N * sizeof(float));
+ float *c = (float *)(ram + 2 * N * sizeof(float));
+ for (int i = 0; i < N; ++i) {
+ a[i] = float(i + 1);
+ b[i] = float(i + 1);
+ c[i] = 0.0;
+ }
+For illustration, we allocate a barrier in the shared DSPM.
+// allocate barrier object for synchronization
+ pthread_barrier_t *bar = (pthread_barrier_t *)(dspm);
+ // initialize barrier with participants = 1 manager + NUM_WORKER workers
+ __barrier_init(bar, NUM_WORKER + 1);
+Next, we allocate and spawn the worker threads.
+// allocate thread objects for each "worker" PE
+ pthread_t *workers = new pthread_t[NUM_WORKER];
+// create vector of core IDs to send to each thread
+ unsigned *tids = new unsigned[NUM_WORKER];
+ for (int i = 0; i < NUM_WORKER; ++i) {
+ tids[i] = i + 1;
+ // spawn worker thread
+ pthread_create(workers + i, NULL, work, &tids[i]);
+ }
+The most important part of the code for the manager -- distribute and push work to the workers!
+// partition the work and push work "packets"
+ for (int i = 0; i < NUM_WORKER; ++i) {
+ // each worker is assigned floor(N / NUM_WORKER) elements
+ int n = N / NUM_WORKER;
+ int start_idx = i * n;
+ int end_idx = (i + 1) * n - 1;
+
+ // handle trailing elements by assigning to final worker
+ if (i == NUM_WORKER - 1) {
+ end_idx = N - 1;
+ }
+// push through work queues
+ __push(i + 1, (uintptr_t)(a));
+ __push(i + 1, (uintptr_t)(b));
+ __push(i + 1, (uintptr_t)(c));
+ __push(i + 1, (unsigned)(start_idx));
+ __push(i + 1, (unsigned)(end_idx));
+ __push(i + 1, (uintptr_t)(bar));
+ }
+// ----- ROI begin -----
+ __reset_stats(); // begin recording time here
+ for (int i = 0; i < NUM_WORKER; ++i) {
+ __push(i + 1, 0); // start signal, value is ignored
+ }
+ __barrier_wait(bar); // synchronize with worker threads
+
+ __dump_reset_stats(); // end recording time here
+// ----- ROI end -----
+// join with all threads
+ for (int tid = 0; tid < NUM_WORKER; ++tid) {
+ pthread_join(workers[tid], NULL);
+ }
+// clean up
+#ifdef EMULATION
+ munmap(dspm, SPM_SIZE_BYTES);
+#endif // EMULATION
+ munmap(ram, RAM_SIZE_BYTES);
+ delete[] workers;
+ delete[] tids;
+ __teardown_queues();
+} // end of main()
+work()¶void *work(void *arg) {
+ unsigned tid = *(unsigned *)(arg);
+ __register_core_id(tid);
+
+ // retrieve variables from work queue
+ volatile float *a = (volatile float *)__pop(0);
+ volatile float *b = (volatile float *)__pop(0);
+ volatile float *c = (volatile float *)__pop(0);
+ int start_idx = (int)__pop(0);
+ int end_idx = (int)__pop(0);
+ pthread_barrier_t *bar = (pthread_barrier_t *)__pop(0);
+// receive start signal
+ __pop(0);
+
+ // perform actual computation
+ for (int i = start_idx; i <= end_idx; ++i) {
+ c[i] += a[i] + b[i];
+ }
+
+ // synchronize with manager
+ __barrier_wait(bar);
+
+ return NULL;
+} // end of work()
+# build emulator library
+%cd emu
+!mkdir -p build
+%cd build
+!cmake .. > /dev/null && make
+%cd ../..
+# build application with emulation library
+%cd example/app
+%rm -rf build
+%mkdir -p build
+%cd build
+!CC=/usr/bin/gcc CXX=/usr/bin/g++ MODE=EMU cmake .. > /dev/null && make
+%cd ../../..
+# run emulated application for functional verification
+%cd example/app/build
+!./vector_add
+%cd ../../..
+# build application with emulation library
+%cd example/app
+%rm -rf build
+%mkdir -p build
+%cd build
+!MODE=SIM cmake .. > /dev/null && make
+%cd ../../../
+# run gem5 simulation
+%cd scripts
+!MODE=SIM APP=vector_add bash run-gem5.sh > /dev/null
+!echo "..." && tail -n20 ../gem5/m5out/run.log
+%cd ../
+### Gather Runtime
+%mkdir -p res
+%cp gem5/m5out/stats.txt res/stats.det.txt
+!grep sim_ticks res/stats.det.txt | head -n1 | tr -s ' ' | cut -d' ' -f2 > ticks.det.txt
+We will first make the required changes to enable tracing in the CPP program that we wrote and then run the trace generation step.
+ +#if defined(AUTO_TRACING) || defined(MANUAL_TRACING)
+#include "hetsim_default_rt.h"
+#endif
+
+void *work(void *arg) {
+ unsigned tid = *(unsigned *)(arg);
+ __register_core_id(tid);
+#if defined(AUTO_TRACING) || defined(MANUAL_TRACING)
+ __open_trace_log(tid);
+#endif // AUTO_TRACING || MANUAL_TRACING
+ // ROI begin
+ ...
+ // ROI end
+#if defined(AUTO_TRACING) || defined(MANUAL_TRACING)
+ __close_trace_log(tid);
+#endif // AUTO_TRACING || MANUAL_TRACING
+}
+int main() {
+ ...
+#if defined(AUTO_TRACING) || defined(MANUAL_TRACING)
+ __open_trace_log(0); // use core-id as argument
+#endif
+ ...
+ __barrier_init(bar, NUM_WORKER + 1); // set number of participants to 1 manager + NUM_WORKER worker PEs
+ ...
+ __dump_reset_stats(); // end recording time here
+ // ----- ROI end -----
+
+#if defined(AUTO_TRACING) || defined(MANUAL_TRACING)
+ __close_trace_log(0);
+#endif // AUTO_TRACING || MANUAL_TRACING
+ ...
+} // end of main()
+# run trace generation
+%cd example/app/build
+!mkdir -p traces
+%rm -f CMakeCache.txt
+!MODE=EMU_AUTO_TRACE cmake .. > /dev/null && make
+!./vector_add
+### Trace Format
+!head -n10 traces/pe_[01].trace
+%cd ../../../
+We will now run the generated traces through the TRE-enabled gem5 model.
%cd scripts
+!MODE=EMU_TRACE APP=vector_add bash run-gem5.sh > /dev/null
+!echo "..." && tail -n20 ../gem5/m5out/run.log
+%cd ..
+As the final step, we will compare the runtime between the detailed gem5 run and the TRE-enabled gem5 run.
%cp gem5/m5out/stats.txt res/stats.tre.txt
+!grep sim_ticks res/stats.tre.txt | head -n1 | tr -s ' ' | cut -d' ' -f2 > ticks.tre.txt
+!cat ticks.det.txt
+!cat ticks.tre.txt
++ +This Jupyter Notebook is available in the public repository: https://github.com/umich-cadre/HetSim-gem5/demos/iiswc-20/tutorial.ipynb
+