diff --git a/benchmark/benchmark.c b/benchmark/benchmark.c index 1eae9056c..abd4ac5d5 100644 --- a/benchmark/benchmark.c +++ b/benchmark/benchmark.c @@ -3,6 +3,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include #include #include #include @@ -15,39 +16,38 @@ #include #include #include +#include + +#define SERIAL_TX_CH 0 + +char *serial_tx_data; +serial_queue_t *serial_tx_queue; +serial_queue_handle_t serial_tx_queue_handle; #define LOG_BUFFER_CAP 7 -/* Notification channels and TCB CAP offsets - ensure these align with .system file! */ -#define START 1 -#define STOP 2 -#define INIT 3 +/* Channel START signal is received on. */ +#define RX_START 1 -#define PD_TOTAL 0 -#define PD_ETH_ID 1 -#define PD_VIRT_RX_ID 2 -#define PD_VIRT_TX_ID 3 -#define PD_COPY_ID 4 -#define PD_COPY1_ID 5 -#define PD_LWIP_ID 6 -#define PD_LWIP1_ID 7 -#define PD_TIMER_ID 8 +/* Channel STOP signal is received on. */ +#define RX_STOP 2 -uintptr_t uart_base; -uintptr_t cyclecounters_vaddr; +/* Channel to transmit START signal. */ +#define TX_START 3 -ccnt_t counter_values[8]; -counter_bitfield_t benchmark_bf; +/* Channel to transmit STOP signal. */ +#define TX_STOP 4 -#define SERIAL_TX_CH 0 +/* Channel to initialise idle thread. */ +#define IDLE_INIT 5 -char *serial_tx_data; -serial_queue_t *serial_tx_queue; -serial_queue_handle_t serial_tx_queue_handle; +/* Identifier used to track utilisation for entire core. */ +#define TOTAL_ID 0 -#ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES -benchmark_track_kernel_entry_t *log_buffer; -#endif +core_config_t core_config; + +ccnt_t counter_values[8]; +counter_bitfield_t benchmark_bf; char *counter_names[] = { "L1 i-cache misses", @@ -67,95 +67,58 @@ event_id_t benchmarking_events[] = { SEL4BENCH_EVENT_BRANCH_MISPREDICT, }; -static void print_pdid_name(uint64_t pd_id) -{ - switch (pd_id) { - case PD_ETH_ID: - sddf_printf(NET_DRIVER_NAME); - break; - case PD_VIRT_RX_ID: - sddf_printf(NET_VIRT_RX_NAME); - break; - case PD_VIRT_TX_ID: - sddf_printf(NET_VIRT_TX_NAME); - break; - case PD_COPY_ID: - sddf_printf(NET_COPY0_NAME); - break; - case PD_COPY1_ID: - sddf_printf(NET_COPY1_NAME); - break; - case PD_LWIP_ID: - sddf_printf(NET_CLI0_NAME); - break; - case PD_LWIP1_ID: - sddf_printf(NET_CLI1_NAME); - break; - case PD_TIMER_ID: - sddf_printf(NET_TIMER_NAME); - break; - default: - sddf_printf("unknown"); - break; - } -} - #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION -static void microkit_benchmark_start(void) +static void benchmark_start_core() { seL4_BenchmarkResetThreadUtilisation(TCB_CAP); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_ETH_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_VIRT_RX_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_VIRT_TX_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_COPY_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_COPY1_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_LWIP_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_LWIP1_ID); - seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + PD_TIMER_ID); + for (uint32_t id = 1; id < core_config.max_core_id + 1; id++) { + if (core_config.core_bitmap & (1 << id)) { + seL4_BenchmarkResetThreadUtilisation(BASE_TCB_CAP + id); + } + } seL4_BenchmarkResetLog(); } -static void microkit_benchmark_stop(uint64_t *total, uint64_t *number_schedules, uint64_t *kernel, uint64_t *entries) +static void benchmark_print_IPC_data(uint64_t *buffer, uint32_t id) { - seL4_BenchmarkFinalizeLog(); - seL4_BenchmarkGetThreadUtilisation(TCB_CAP); - uint64_t *buffer = (uint64_t *)&seL4_GetIPCBuffer()->msg[0]; - - *total = buffer[BENCHMARK_TOTAL_UTILISATION]; - *number_schedules = buffer[BENCHMARK_TOTAL_NUMBER_SCHEDULES]; - *kernel = buffer[BENCHMARK_TOTAL_KERNEL_UTILISATION]; - *entries = buffer[BENCHMARK_TOTAL_NUMBER_KERNEL_ENTRIES]; + uint64_t total, number_schedules, kernel, entries; + if (id == TOTAL_ID) { + total = buffer[BENCHMARK_TOTAL_UTILISATION]; + number_schedules = buffer[BENCHMARK_TOTAL_NUMBER_SCHEDULES]; + kernel = buffer[BENCHMARK_TOTAL_KERNEL_UTILISATION]; + entries = buffer[BENCHMARK_TOTAL_NUMBER_KERNEL_ENTRIES]; + sddf_printf("Total utilisation details: \n"); + } else { + total = buffer[BENCHMARK_TCB_UTILISATION]; + number_schedules = buffer[BENCHMARK_TCB_NUMBER_SCHEDULES]; + kernel = buffer[BENCHMARK_TCB_KERNEL_UTILISATION]; + entries = buffer[BENCHMARK_TCB_NUMBER_KERNEL_ENTRIES]; + sddf_printf("Utilisation details for PD: %s (%x)\n", pd_id_to_name(id), id); + } + sddf_printf("{\nKernelUtilisation: %lx\nKernelEntries: " + "%lx\nNumberSchedules: %lx\nTotalUtilisation: %lx\n}\n", + kernel, entries, number_schedules, total); } -static void microkit_benchmark_stop_tcb(uint64_t pd_id, uint64_t *total, uint64_t *number_schedules, uint64_t *kernel, - uint64_t *entries) +static void benchmark_stop_core() { - seL4_BenchmarkGetThreadUtilisation(BASE_TCB_CAP + pd_id); - uint64_t *buffer = (uint64_t *)&seL4_GetIPCBuffer()->msg[0]; - - *total = buffer[BENCHMARK_TCB_UTILISATION]; - *number_schedules = buffer[BENCHMARK_TCB_NUMBER_SCHEDULES]; - *kernel = buffer[BENCHMARK_TCB_KERNEL_UTILISATION]; - *entries = buffer[BENCHMARK_TCB_NUMBER_KERNEL_ENTRIES]; -} + seL4_BenchmarkFinalizeLog(); + seL4_BenchmarkGetThreadUtilisation(TCB_CAP); + benchmark_print_IPC_data((uint64_t *)&seL4_GetIPCBuffer()->msg[0], TOTAL_ID); -static void print_benchmark_details(uint64_t pd_id, uint64_t kernel_util, uint64_t kernel_entries, - uint64_t number_schedules, uint64_t total_util) -{ - if (pd_id == PD_TOTAL) { - sddf_printf("Total utilisation details: \n"); - } else { - sddf_printf("Utilisation details for PD: "); - print_pdid_name(pd_id); - sddf_printf(" (%lx)\n", pd_id); + for (uint32_t id = 1; id < core_config.max_core_id + 1; id++) { + if (core_config.core_bitmap & (1 << id)) { + seL4_BenchmarkGetThreadUtilisation(BASE_TCB_CAP + id); + benchmark_print_IPC_data((uint64_t *)&seL4_GetIPCBuffer()->msg[0], id); + } } - sddf_printf("{\nKernelUtilisation: %lx\nKernelEntries: %lx\nNumberSchedules: %lx\nTotalUtilisation: %lx\n}\n", - kernel_util, kernel_entries, number_schedules, total_util); } #endif #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES -static inline void seL4_BenchmarkTrackDumpSummary(benchmark_track_kernel_entry_t *logBuffer, uint64_t logSize) +benchmark_track_kernel_entry_t *log_buffer; + +static inline void seL4_BenchmarkTrackDumpSummary(uint64_t logSize) { seL4_Word index = 0; seL4_Word syscall_entries = 0; @@ -166,19 +129,19 @@ static inline void seL4_BenchmarkTrackDumpSummary(benchmark_track_kernel_entry_t seL4_Word debug_fault = 0; seL4_Word other = 0; - while (logBuffer[index].start_time != 0 && index < logSize) { - if (logBuffer[index].entry.path == Entry_Syscall) { - if (logBuffer[index].entry.is_fastpath) { + while (log_buffer[index].start_time != 0 && index < logSize) { + if (log_buffer[index].entry.path == Entry_Syscall) { + if (log_buffer[index].entry.is_fastpath) { fastpaths++; } syscall_entries++; - } else if (logBuffer[index].entry.path == Entry_Interrupt) { + } else if (log_buffer[index].entry.path == Entry_Interrupt) { interrupt_entries++; - } else if (logBuffer[index].entry.path == Entry_UserLevelFault) { + } else if (log_buffer[index].entry.path == Entry_UserLevelFault) { userlevelfault_entries++; - } else if (logBuffer[index].entry.path == Entry_VMFault) { + } else if (log_buffer[index].entry.path == Entry_VMFault) { vmfault_entries++; - } else if (logBuffer[index].entry.path == Entry_DebugFault) { + } else if (log_buffer[index].entry.path == Entry_DebugFault) { debug_fault++; } else { other++; @@ -187,12 +150,13 @@ static inline void seL4_BenchmarkTrackDumpSummary(benchmark_track_kernel_entry_t index++; } - sddf_printf("Number of system call invocations %llx and fastpaths %llx\n", syscall_entries, fastpaths); - sddf_printf("Number of interrupt invocations %llx\n", interrupt_entries); - sddf_printf("Number of user-level faults %llx\n", userlevelfault_entries); - sddf_printf("Number of VM faults %llx\n", vmfault_entries); - sddf_printf("Number of debug faults %llx\n", debug_fault); - sddf_printf("Number of others %llx\n", other); + sddf_printf("System call invocations %lx", syscall_entries); + sddf_printf("Fastpaths %lx\n", fastpaths); + sddf_printf("Interrupt invocations %lx\n", interrupt_entries); + sddf_printf("User-level faults %lx\n", userlevelfault_entries); + sddf_printf("VM faults %lx\n", vmfault_entries); + sddf_printf("Debug faults %lx\n", debug_fault); + sddf_printf("Others %lx\n", other); } #endif @@ -200,28 +164,30 @@ static inline void seL4_BenchmarkTrackDumpSummary(benchmark_track_kernel_entry_t void notified(microkit_channel ch) { switch (ch) { - case START: + case RX_START: #ifdef MICROKIT_CONFIG_benchmark sel4bench_reset_counters(); THREAD_MEMORY_RELEASE(); sel4bench_start_counters(benchmark_bf); #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION - microkit_benchmark_start(); + benchmark_start_core(); #endif #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES seL4_BenchmarkResetLog(); #endif #endif - + if (!core_config.last_core) + microkit_notify(TX_START); break; - case STOP: + + case RX_STOP: #ifdef MICROKIT_CONFIG_benchmark sel4bench_get_counters(benchmark_bf, &counter_values[0]); sel4bench_stop_counters(benchmark_bf); - sddf_printf("{\n"); + sddf_printf("{CORE %u: \n", core_config.core_value); for (int i = 0; i < ARRAY_SIZE(benchmarking_events); i++) { sddf_printf("%s: %lX\n", counter_names[i], counter_values[i]); } @@ -229,63 +195,41 @@ void notified(microkit_channel ch) #endif #ifdef CONFIG_BENCHMARK_TRACK_UTILISATION - uint64_t total; - uint64_t kernel; - uint64_t entries; - uint64_t number_schedules; - microkit_benchmark_stop(&total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_TOTAL, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_ETH_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_ETH_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_VIRT_RX_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_VIRT_RX_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_VIRT_TX_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_VIRT_TX_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_COPY_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_COPY_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_COPY1_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_COPY1_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_LWIP_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_LWIP_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_LWIP1_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_LWIP1_ID, kernel, entries, number_schedules, total); - - microkit_benchmark_stop_tcb(PD_TIMER_ID, &total, &number_schedules, &kernel, &entries); - print_benchmark_details(PD_TIMER_ID, kernel, entries, number_schedules, total); + benchmark_stop_core(); #endif #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES - entries = seL4_BenchmarkFinalizeLog(); - sddf_printf("KernelEntries: %llx\n", entries); - seL4_BenchmarkTrackDumpSummary(log_buffer, entries); + uint64_t log_size = seL4_BenchmarkFinalizeLog(); + sddf_printf("KernelEntries: %lx\n", log_size); + seL4_BenchmarkTrackDumpSummary(log_size); #endif - + THREAD_MEMORY_RELEASE(); + if (!core_config.last_core) + microkit_notify(TX_STOP); break; + case SERIAL_TX_CH: // Nothing to do break; + default: - sddf_printf("Bench thread notified on unexpected channel\n"); + sddf_printf("BENCH|LOG: Bench thread notified on unexpected channel\n"); } } void init(void) { - serial_cli_queue_init_sys(microkit_name, NULL, NULL, NULL, &serial_tx_queue_handle, serial_tx_queue, serial_tx_data); + serial_cli_queue_init_sys(microkit_name, NULL, NULL, NULL, &serial_tx_queue_handle, serial_tx_queue, + serial_tx_data); serial_putchar_init(SERIAL_TX_CH, &serial_tx_queue_handle); + + bench_core_config_info(microkit_name, &core_config); + #ifdef MICROKIT_CONFIG_benchmark sel4bench_init(); - seL4_Word n_counters = sel4bench_get_num_counters(); + seL4_Word n_counters = sel4bench_get_num_counters(); counter_bitfield_t mask = 0; - for (seL4_Word counter = 0; counter < n_counters; counter++) { if (counter >= ARRAY_SIZE(benchmarking_events)) { break; @@ -301,28 +245,26 @@ void init(void) sddf_dprintf("BENCH|LOG: Bench running in debug mode, no access to counters\n"); #endif - /* Notify the idle thread that the sel4bench library is initialised. */ - microkit_notify(INIT); + microkit_notify(IDLE_INIT); #ifdef CONFIG_BENCHMARK_TRACK_KERNEL_ENTRIES int res_buf = seL4_BenchmarkSetLogBuffer(LOG_BUFFER_CAP); if (res_buf) { - sddf_printf("Could not set log buffer: %llx\n", res_buf); + sddf_printf("BENCH|ERROR: Could not set log buffer: %d\n", res_buf); } else { - sddf_printf("Log buffer set\n"); + sddf_printf("BENCH|LOG: Log buffer set\n"); } #endif } seL4_Bool fault(microkit_child id, microkit_msginfo msginfo, microkit_msginfo *reply_msginfo) { - sddf_printf("BENCH|LOG: Faulting PD "); - print_pdid_name(id); - sddf_printf(" (%x)\n", id); + sddf_printf("BENCH|LOG: Faulting PD %s (%x)\n", pd_id_to_name(id), id); seL4_UserContext regs; seL4_TCB_ReadRegisters(BASE_TCB_CAP + id, false, 0, sizeof(seL4_UserContext) / sizeof(seL4_Word), ®s); - sddf_printf("Registers: \npc : %lx\nspsr : %lx\nx0 : %lx\nx1 : %lx\nx2 : %lx\nx3 : %lx\nx4 : %lx\nx5 : %lx\nx6 : %lx\nx7 : %lx\n", + sddf_printf("Registers: \npc : %lx\nspsr : %lx\nx0 : %lx\nx1 : %lx\n" + "x2 : %lx\nx3 : %lx\nx4 : %lx\nx5 : %lx\nx6 : %lx\nx7 : %lx\n", regs.pc, regs.spsr, regs.x0, regs.x1, regs.x2, regs.x3, regs.x4, regs.x5, regs.x6, regs.x7); switch (microkit_msginfo_get_label(msginfo)) { @@ -353,4 +295,4 @@ seL4_Bool fault(microkit_child id, microkit_msginfo msginfo, microkit_msginfo *r } return seL4_False; -} \ No newline at end of file +} diff --git a/benchmark/idle.c b/benchmark/idle.c index ad30f4180..7907e0c3f 100644 --- a/benchmark/idle.c +++ b/benchmark/idle.c @@ -13,7 +13,6 @@ #define INIT 3 #define MAGIC_CYCLES 150 -uintptr_t cyclecounters_vaddr; struct bench *b; void count_idle(void) @@ -48,6 +47,5 @@ void notified(microkit_channel ch) void init(void) { - b = (void *)cyclecounters_vaddr; return; -} \ No newline at end of file +} diff --git a/examples/echo_server/Makefile b/examples/echo_server/Makefile index e7376ac71..432702ca8 100644 --- a/examples/echo_server/Makefile +++ b/examples/echo_server/Makefile @@ -24,11 +24,23 @@ ifeq ($(strip $(MICROKIT_BOARD)), odroidc4) export UART_DRIV_DIR := meson export TIMER_DRV_DIR := meson export CPU := cortex-a55 +else ifeq ($(strip $(MICROKIT_BOARD)), odroidc4_4_cores) + export DRIV_DIR := meson + export UART_DRIV_DIR := meson + export TIMER_DRV_DIR := meson + export CPU := cortex-a55 + export CONFIG_INCLUDE_SMP := _smp else ifneq ($(filter $(strip $(MICROKIT_BOARD)),imx8mm_evk imx8mp_evk maaxboard),) export DRIV_DIR := imx export UART_DRIV_DIR := imx export TIMER_DRV_DIR := imx export CPU := cortex-a53 +else ifeq ($(strip $(MICROKIT_BOARD)), imx8mm_evk_4_cores) + export DRIV_DIR := imx + export UART_DRIV_DIR := imx + export TIMER_DRV_DIR := imx + export CPU := cortex-a53 + export CONFIG_INCLUDE_SMP := _smp else ifeq ($(strip $(MICROKIT_BOARD)), qemu_virt_aarch64) export DRIV_DIR := virtio export UART_DRIV_DIR := arm diff --git a/examples/echo_server/board/imx8mm_evk/echo_server.system b/examples/echo_server/board/imx8mm_evk/echo_server.system index 264e4e867..8972117a6 100644 --- a/examples/echo_server/board/imx8mm_evk/echo_server.system +++ b/examples/echo_server/board/imx8mm_evk/echo_server.system @@ -60,10 +60,10 @@ - + - + @@ -177,7 +177,7 @@ - + @@ -219,7 +219,7 @@ - + @@ -264,17 +264,17 @@ - + - + - + diff --git a/examples/echo_server/board/imx8mm_evk_4_cores/echo_server.system b/examples/echo_server/board/imx8mm_evk_4_cores/echo_server.system new file mode 100644 index 000000000..ab03df33c --- /dev/null +++ b/examples/echo_server/board/imx8mm_evk_4_cores/echo_server.system @@ -0,0 +1,403 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ethernet interrupt --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/echo_server/board/imx8mp_evk/echo_server.system b/examples/echo_server/board/imx8mp_evk/echo_server.system index 264e4e867..8972117a6 100644 --- a/examples/echo_server/board/imx8mp_evk/echo_server.system +++ b/examples/echo_server/board/imx8mp_evk/echo_server.system @@ -60,10 +60,10 @@ - + - + @@ -177,7 +177,7 @@ - + @@ -219,7 +219,7 @@ - + @@ -264,17 +264,17 @@ - + - + - + diff --git a/examples/echo_server/board/maaxboard/echo_server.system b/examples/echo_server/board/maaxboard/echo_server.system index 05e1d7684..acdf5ed7f 100644 --- a/examples/echo_server/board/maaxboard/echo_server.system +++ b/examples/echo_server/board/maaxboard/echo_server.system @@ -60,10 +60,10 @@ - + - + @@ -177,7 +177,7 @@ - + @@ -219,7 +219,7 @@ - + @@ -264,17 +264,17 @@ - + - + - + diff --git a/examples/echo_server/board/odroidc4/echo_server.system b/examples/echo_server/board/odroidc4/echo_server.system index 74998d981..d0cb78834 100644 --- a/examples/echo_server/board/odroidc4/echo_server.system +++ b/examples/echo_server/board/odroidc4/echo_server.system @@ -60,10 +60,10 @@ - + - + @@ -177,7 +177,7 @@ - + @@ -219,7 +219,7 @@ - + @@ -264,17 +264,17 @@ - + - + - + diff --git a/examples/echo_server/board/odroidc4_4_cores/echo_server.system b/examples/echo_server/board/odroidc4_4_cores/echo_server.system new file mode 100644 index 000000000..5636da798 --- /dev/null +++ b/examples/echo_server/board/odroidc4_4_cores/echo_server.system @@ -0,0 +1,403 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ethernet interrupt --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/echo_server/board/qemu_virt_aarch64/echo_server.system b/examples/echo_server/board/qemu_virt_aarch64/echo_server.system index 1cacbb3c9..c2cb5a834 100644 --- a/examples/echo_server/board/qemu_virt_aarch64/echo_server.system +++ b/examples/echo_server/board/qemu_virt_aarch64/echo_server.system @@ -59,10 +59,10 @@ - + - + @@ -176,7 +176,7 @@ - + @@ -217,7 +217,7 @@ - + @@ -262,17 +262,17 @@ - + - + - + diff --git a/examples/echo_server/echo.mk b/examples/echo_server/echo.mk index bf5e3a67c..fa9add691 100644 --- a/examples/echo_server/echo.mk +++ b/examples/echo_server/echo.mk @@ -7,19 +7,24 @@ QEMU := qemu-system-aarch64 MICROKIT_TOOL ?= $(MICROKIT_SDK)/bin/microkit -ECHO_SERVER:=${SDDF}/examples/echo_server -LWIPDIR:=network/ipstacks/lwip/src +BOARD_DIR := $(MICROKIT_SDK)/board/$(MICROKIT_BOARD)/$(MICROKIT_CONFIG) +ECHO_SERVER:=$(SDDF)/examples/echo_server + BENCHMARK:=$(SDDF)/benchmark -UTIL:=$(SDDF)/util +CORE_CONFIG_INCLUDE:=$(ECHO_SERVER)/include/core_config$(CONFIG_INCLUDE_SMP) + +LWIPDIR:=network/ipstacks/lwip/src +NETWORK_COMPONENTS:=$(SDDF)/network/components ETHERNET_DRIVER:=$(SDDF)/drivers/network/$(DRIV_DIR) -ETHERNET_CONFIG_INCLUDE:=${ECHO_SERVER}/include/ethernet_config +ETHERNET_CONFIG_INCLUDE:=$(ECHO_SERVER)/include/ethernet_config + SERIAL_COMPONENTS := $(SDDF)/serial/components UART_DRIVER := $(SDDF)/drivers/serial/$(UART_DRIV_DIR) -SERIAL_CONFIG_INCLUDE:=${ECHO_SERVER}/include/serial_config +SERIAL_CONFIG_INCLUDE:=$(ECHO_SERVER)/include/serial_config$(CONFIG_INCLUDE_SMP) + TIMER_DRIVER:=$(SDDF)/drivers/timer/$(TIMER_DRV_DIR) -NETWORK_COMPONENTS:=$(SDDF)/network/components +UTIL:=$(SDDF)/util -BOARD_DIR := $(MICROKIT_SDK)/board/$(MICROKIT_BOARD)/$(MICROKIT_CONFIG) SYSTEM_FILE := ${ECHO_SERVER}/board/$(MICROKIT_BOARD)/echo_server.system IMAGE_FILE := loader.img REPORT_FILE := report.txt @@ -37,11 +42,12 @@ CFLAGS := -mcpu=$(CPU) \ -DMICROKIT_CONFIG_$(MICROKIT_CONFIG) \ -I$(BOARD_DIR)/include \ -I$(SDDF)/include \ - -I${ECHO_INCLUDE}/lwip \ - -I${ETHERNET_CONFIG_INCLUDE} \ + -I$(ECHO_INCLUDE)/lwip \ + -I$(CORE_CONFIG_INCLUDE) \ + -I$(ETHERNET_CONFIG_INCLUDE) \ -I$(SERIAL_CONFIG_INCLUDE) \ - -I${SDDF}/$(LWIPDIR)/include \ - -I${SDDF}/$(LWIPDIR)/include/ipv4 \ + -I$(SDDF)/$(LWIPDIR)/include \ + -I$(SDDF)/$(LWIPDIR)/include/ipv4 \ -MD \ -MP diff --git a/examples/echo_server/include/core_config/core_config.h b/examples/echo_server/include/core_config/core_config.h new file mode 100644 index 000000000..5f3748b08 --- /dev/null +++ b/examples/echo_server/include/core_config/core_config.h @@ -0,0 +1,97 @@ +/* + * Copyright 2024, UNSW + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +/* Total number of cores on the board. */ +#define MAX_CORES 4 + +/* Total number of PDs being tracked during the benchmark. */ +#define NUM_PDS_TO_TRACK 8 + +/* The core of the benchmark PD which is last in the notification chain. + The benchmark PD on this core receives START and STOP notifications, + but does not send them. */ +#define LAST_CORE 0 + +typedef struct { + uint16_t pd_id; /* ID of the PD. */ + uint16_t pd_core; /* Core of the PD. */ + char *pd_name; /* Name of the PD. */ +} pd_core_info_t; + +pd_core_info_t pd_core_info[NUM_PDS_TO_TRACK] = { + { 1, 0, NET_DRIVER_NAME }, /* Ethernet driver. */ + { 2, 0, NET_VIRT_RX_NAME }, /* Rx virtualiser. */ + { 3, 0, NET_VIRT_TX_NAME }, /* Tx virtualiser. */ + { 4, 0, NET_COPY0_NAME }, /* Copy component 0. */ + { 5, 0, NET_COPY1_NAME }, /* Copy component 1. */ + { 6, 0, NET_CLI0_NAME }, /* LWIP client 0. */ + { 7, 0, NET_CLI1_NAME }, /* LWIP client 1. */ + { 8, 0, NET_TIMER_NAME } /* Timer driver. */ +}; + +typedef struct { + uint64_t core_bitmap; /* Bitmap of client IDs on this core. */ + uint32_t core_value; /* Core of this benchmark process. */ + uint32_t max_core_id; /* Maximum ID of client on this core. */ + bool last_core; /* Whether this bench process is the last in the notification chain. */ +} core_config_t; + +static void bench_core_config_info(char *pd_name, core_config_t *core_config) +{ + core_config->max_core_id = 0; + core_config->last_core = false; + + if (!sddf_strcmp(pd_name, "bench0")) { + core_config->core_value = 0; + } else if (!sddf_strcmp(pd_name, "bench1")) { + core_config->core_value = 1; + } else if (!sddf_strcmp(pd_name, "bench2")) { + core_config->core_value = 2; + } else if (!sddf_strcmp(pd_name, "bench3")) { + core_config->core_value = 3; + } else { + return; + } + + if (core_config->core_value == LAST_CORE) { + core_config->last_core = true; + } + + for (uint16_t i = 0; i < NUM_PDS_TO_TRACK; i++) { + if (pd_core_info[i].pd_core == core_config->core_value) { + core_config->core_bitmap |= (1 << pd_core_info[i].pd_id); + if (pd_core_info[i].pd_id > core_config->max_core_id) { + core_config->max_core_id = pd_core_info[i].pd_id; + } + } + } +} + +static uint16_t bench_active_cores() +{ + uint16_t active_cores = 0; + for (uint16_t i = 0; i < NUM_PDS_TO_TRACK; i++) { + active_cores |= (1 << pd_core_info[i].pd_core); + } + + return active_cores; +} + +static char *pd_id_to_name(uint16_t pd_id) +{ + for (uint16_t i = 0; i < NUM_PDS_TO_TRACK; i++) { + if (pd_core_info[i].pd_id == pd_id) { + return pd_core_info[i].pd_name; + } + } + + return NULL; +} diff --git a/examples/echo_server/include/core_config_smp/core_config.h b/examples/echo_server/include/core_config_smp/core_config.h new file mode 100644 index 000000000..de28138bc --- /dev/null +++ b/examples/echo_server/include/core_config_smp/core_config.h @@ -0,0 +1,97 @@ +/* + * Copyright 2024, UNSW + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +/* Total number of cores on the board. */ +#define MAX_CORES 4 + +/* Total number of PDs being tracked during the benchmark. */ +#define NUM_PDS_TO_TRACK 8 + +/* The core of the benchmark PD which is last in the notification chain. + The benchmark PD on this core receives START and STOP notifications, + but does not send them. */ +#define LAST_CORE 1 + +typedef struct { + uint16_t pd_id; /* ID of the PD. */ + uint16_t pd_core; /* Core of the PD. */ + char *pd_name; /* Name of the PD. */ +} pd_core_info_t; + +pd_core_info_t pd_core_info[NUM_PDS_TO_TRACK] = { + { 1, 0, NET_DRIVER_NAME }, /* Ethernet driver. */ + { 2, 1, NET_VIRT_RX_NAME }, /* Rx virtualiser. */ + { 3, 0, NET_VIRT_TX_NAME }, /* Tx virtualiser. */ + { 4, 1, NET_COPY0_NAME }, /* Copy component 0. */ + { 5, 1, NET_COPY1_NAME }, /* Copy component 1. */ + { 6, 1, NET_CLI0_NAME }, /* LWIP client 0. */ + { 7, 1, NET_CLI1_NAME }, /* LWIP client 1. */ + { 8, 1, NET_TIMER_NAME } /* Timer driver. */ +}; + +typedef struct { + uint64_t core_bitmap; /* Bitmap of client IDs on this core. */ + uint32_t core_value; /* Core of this benchmark process. */ + uint32_t max_core_id; /* Maximum ID of client on this core. */ + bool last_core; /* Whether this bench process is the last in the notification chain. */ +} core_config_t; + +static void bench_core_config_info(char *pd_name, core_config_t *core_config) +{ + core_config->max_core_id = 0; + core_config->last_core = false; + + if (!sddf_strcmp(pd_name, "bench0")) { + core_config->core_value = 0; + } else if (!sddf_strcmp(pd_name, "bench1")) { + core_config->core_value = 1; + } else if (!sddf_strcmp(pd_name, "bench2")) { + core_config->core_value = 2; + } else if (!sddf_strcmp(pd_name, "bench3")) { + core_config->core_value = 3; + } else { + return; + } + + if (core_config->core_value == LAST_CORE) { + core_config->last_core = true; + } + + for (uint16_t i = 0; i < NUM_PDS_TO_TRACK; i++) { + if (pd_core_info[i].pd_core == core_config->core_value) { + core_config->core_bitmap |= (1 << pd_core_info[i].pd_id); + if (pd_core_info[i].pd_id > core_config->max_core_id) { + core_config->max_core_id = pd_core_info[i].pd_id; + } + } + } +} + +static uint16_t bench_active_cores() +{ + uint16_t active_cores = 0; + for (uint16_t i = 0; i < NUM_PDS_TO_TRACK; i++) { + active_cores |= (1 << pd_core_info[i].pd_core); + } + + return active_cores; +} + +static char *pd_id_to_name(uint16_t pd_id) +{ + for (uint16_t i = 0; i < NUM_PDS_TO_TRACK; i++) { + if (pd_core_info[i].pd_id == pd_id) { + return pd_core_info[i].pd_name; + } + } + + return NULL; +} diff --git a/examples/echo_server/include/serial_config/serial_config.h b/examples/echo_server/include/serial_config/serial_config.h index cd0e744a0..ebe4626a1 100644 --- a/examples/echo_server/include/serial_config/serial_config.h +++ b/examples/echo_server/include/serial_config/serial_config.h @@ -30,7 +30,7 @@ #define SERIAL_CLI0_NAME "client0" #define SERIAL_CLI1_NAME "client1" -#define SERIAL_CLI2_NAME "bench" +#define SERIAL_CLI2_NAME "bench0" #define SERIAL_VIRT_TX_NAME "serial_virt_tx" #define SERIAL_QUEUE_SIZE 0x1000 diff --git a/examples/echo_server/include/serial_config_smp/serial_config.h b/examples/echo_server/include/serial_config_smp/serial_config.h new file mode 100644 index 000000000..2a498e530 --- /dev/null +++ b/examples/echo_server/include/serial_config_smp/serial_config.h @@ -0,0 +1,121 @@ +/* + * Copyright 2024, UNSW + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +/* Number of clients that can be connected to the serial server. */ +#define SERIAL_NUM_CLIENTS 6 + +/* Only support transmission and not receive. */ +#define SERIAL_TX_ONLY 1 + +/* Associate a colour with each client's output. */ +#define SERIAL_WITH_COLOUR 1 + +/* Control character to switch input stream - ctrl \. To input character input twice. */ +#define SERIAL_SWITCH_CHAR 28 + +/* Control character to terminate client number input. */ +#define SERIAL_TERMINATE_NUM '\r' + +/* Default baud rate of the uart device */ +#define UART_DEFAULT_BAUD 115200 + +#define SERIAL_CLI0_NAME "client0" +#define SERIAL_CLI1_NAME "client1" +#define SERIAL_CLI2_NAME "bench0" +#define SERIAL_CLI3_NAME "bench1" +#define SERIAL_CLI4_NAME "bench2" +#define SERIAL_CLI5_NAME "bench3" +#define SERIAL_VIRT_TX_NAME "serial_virt_tx" + +#define SERIAL_QUEUE_SIZE 0x1000 +#define SERIAL_DATA_REGION_CAPACITY 0x2000 + +#define SERIAL_TX_DATA_REGION_CAPACITY_DRIV (2 * SERIAL_DATA_REGION_CAPACITY) +#define SERIAL_TX_DATA_REGION_CAPACITY_CLI0 SERIAL_DATA_REGION_CAPACITY +#define SERIAL_TX_DATA_REGION_CAPACITY_CLI1 SERIAL_DATA_REGION_CAPACITY +#define SERIAL_TX_DATA_REGION_CAPACITY_CLI2 SERIAL_DATA_REGION_CAPACITY +#define SERIAL_TX_DATA_REGION_CAPACITY_CLI3 SERIAL_DATA_REGION_CAPACITY +#define SERIAL_TX_DATA_REGION_CAPACITY_CLI4 SERIAL_DATA_REGION_CAPACITY +#define SERIAL_TX_DATA_REGION_CAPACITY_CLI5 SERIAL_DATA_REGION_CAPACITY + +/* To avoid deadlocks caused when the virtualiser adds colour codes to the + start and end of strings, driver data region must be larger than any + client data region. */ +#define SERIAL_MAX_CLIENT_TX_DATA_CAPACITY SERIAL_DATA_REGION_CAPACITY +#if SERIAL_WITH_COLOUR +_Static_assert(SERIAL_TX_DATA_REGION_CAPACITY_DRIV > SERIAL_MAX_CLIENT_TX_DATA_CAPACITY, + "Driver TX data region must be larger than all client data regions in SERIAL_WITH_COLOUR mode."); +#endif + +/* Ensure the entire data region can be assigned a unique index by a 32 bit + unsigned. */ +#define SERIAL_MAX_DATA_CAPACITY MAX(SERIAL_TX_DATA_REGION_CAPACITY_DRIV, SERIAL_MAX_CLIENT_TX_DATA_CAPACITY) +_Static_assert(SERIAL_MAX_DATA_CAPACITY < UINT32_MAX, + "Data regions must be smaller than UINT32 max to correctly use queue data structure."); + +static inline void serial_cli_queue_init_sys(char *pd_name, serial_queue_handle_t *rx_queue_handle, + serial_queue_t *rx_queue, char *rx_data, + serial_queue_handle_t *tx_queue_handle, serial_queue_t *tx_queue, + char *tx_data) +{ + if (!sddf_strcmp(pd_name, SERIAL_CLI0_NAME)) { + serial_queue_init(tx_queue_handle, tx_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI0, tx_data); + } else if (!sddf_strcmp(pd_name, SERIAL_CLI1_NAME)) { + serial_queue_init(tx_queue_handle, tx_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI1, tx_data); + } else if (!sddf_strcmp(pd_name, SERIAL_CLI2_NAME)) { + serial_queue_init(tx_queue_handle, tx_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI2, tx_data); + } else if (!sddf_strcmp(pd_name, SERIAL_CLI3_NAME)) { + serial_queue_init(tx_queue_handle, tx_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI3, tx_data); + } else if (!sddf_strcmp(pd_name, SERIAL_CLI4_NAME)) { + serial_queue_init(tx_queue_handle, tx_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI4, tx_data); + } else if (!sddf_strcmp(pd_name, SERIAL_CLI5_NAME)) { + serial_queue_init(tx_queue_handle, tx_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI5, tx_data); + } +} + +static inline void serial_virt_queue_init_sys(char *pd_name, serial_queue_handle_t *cli_queue_handle, + serial_queue_t *cli_queue, char *cli_data) +{ + if (!sddf_strcmp(pd_name, SERIAL_VIRT_TX_NAME)) { + serial_queue_init(cli_queue_handle, cli_queue, SERIAL_TX_DATA_REGION_CAPACITY_CLI0, cli_data); + serial_queue_init(&cli_queue_handle[1], (serial_queue_t *)((uintptr_t)cli_queue + SERIAL_QUEUE_SIZE), + SERIAL_TX_DATA_REGION_CAPACITY_CLI1, cli_data + SERIAL_TX_DATA_REGION_CAPACITY_CLI0); + serial_queue_init(&cli_queue_handle[2], (serial_queue_t *)((uintptr_t)cli_queue + 2 * SERIAL_QUEUE_SIZE), + SERIAL_TX_DATA_REGION_CAPACITY_CLI2, + cli_data + SERIAL_TX_DATA_REGION_CAPACITY_CLI0 + SERIAL_TX_DATA_REGION_CAPACITY_CLI1); + serial_queue_init(&cli_queue_handle[3], (serial_queue_t *)((uintptr_t)cli_queue + 3 * SERIAL_QUEUE_SIZE), + SERIAL_TX_DATA_REGION_CAPACITY_CLI3, + cli_data + SERIAL_TX_DATA_REGION_CAPACITY_CLI0 + SERIAL_TX_DATA_REGION_CAPACITY_CLI1 + + SERIAL_TX_DATA_REGION_CAPACITY_CLI2); + serial_queue_init(&cli_queue_handle[4], (serial_queue_t *)((uintptr_t)cli_queue + 4 * SERIAL_QUEUE_SIZE), + SERIAL_TX_DATA_REGION_CAPACITY_CLI4, + cli_data + SERIAL_TX_DATA_REGION_CAPACITY_CLI0 + SERIAL_TX_DATA_REGION_CAPACITY_CLI1 + + SERIAL_TX_DATA_REGION_CAPACITY_CLI2 + SERIAL_TX_DATA_REGION_CAPACITY_CLI3); + serial_queue_init(&cli_queue_handle[5], (serial_queue_t *)((uintptr_t)cli_queue + 5 * SERIAL_QUEUE_SIZE), + SERIAL_TX_DATA_REGION_CAPACITY_CLI5, + cli_data + SERIAL_TX_DATA_REGION_CAPACITY_CLI0 + SERIAL_TX_DATA_REGION_CAPACITY_CLI1 + + SERIAL_TX_DATA_REGION_CAPACITY_CLI2 + SERIAL_TX_DATA_REGION_CAPACITY_CLI3 + + SERIAL_TX_DATA_REGION_CAPACITY_CLI4); + } +} + +#if SERIAL_WITH_COLOUR +static inline void serial_channel_names_init(char **client_names) +{ + client_names[0] = SERIAL_CLI0_NAME; + client_names[1] = SERIAL_CLI1_NAME; + client_names[2] = SERIAL_CLI2_NAME; + client_names[3] = SERIAL_CLI3_NAME; + client_names[4] = SERIAL_CLI4_NAME; + client_names[5] = SERIAL_CLI5_NAME; +} +#endif diff --git a/examples/echo_server/utilization_socket.c b/examples/echo_server/utilization_socket.c index c69dde1b0..64c9ded72 100644 --- a/examples/echo_server/utilization_socket.c +++ b/examples/echo_server/utilization_socket.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "echo.h" @@ -21,8 +22,6 @@ #define MAX_PACKET_SIZE 0x1000 -uintptr_t cyclecounters_vaddr; - /* This file implements a TCP based utilization measurment process that starts * and stops utilization measurements based on a client's requests. * The protocol used to communicate is as follows: @@ -74,14 +73,16 @@ static struct tcp_pcb *utiliz_socket; "Content-length: "STR(x)"\n"\ ","STR(y)","STR(z) +uintptr_t idle_ccounts_vaddr; -struct bench *bench; +#define IDLE_CCOUNTS_SIZE 0x1000 -uint64_t start; -uint64_t idle_ccount_start; +uint16_t active_cores; -char data_packet_str[MAX_PACKET_SIZE]; +uint64_t core_ccount_start[MAX_CORES]; +uint64_t idle_ccount_start[MAX_CORES]; +char data_packet_str[MAX_PACKET_SIZE]; static inline void my_reverse(char s[]) { @@ -139,8 +140,13 @@ static err_t utilization_recv_callback(void *arg, struct tcp_pcb *pcb, struct pb } else if (msg_match(data_packet_str, START)) { sddf_printf("%s measurement starting... \n", microkit_name); if (!strcmp(microkit_name, "client0")) { - start = __atomic_load_n(&bench->ts, __ATOMIC_RELAXED); - idle_ccount_start = __atomic_load_n(&bench->ccount, __ATOMIC_RELAXED); + for (int i = 0; i < MAX_CORES; i++) { + if (active_cores & (1 << i)) { + struct bench *core_ccounts = (struct bench *)(idle_ccounts_vaddr + IDLE_CCOUNTS_SIZE * i); + core_ccount_start[i] = __atomic_load_n(&core_ccounts->ts, __ATOMIC_RELAXED); + idle_ccount_start[i] = __atomic_load_n(&core_ccounts->ccount, __ATOMIC_RELAXED); + } + } microkit_notify(START_PMU); } } else if (msg_match(data_packet_str, STOP)) { @@ -149,8 +155,13 @@ static err_t utilization_recv_callback(void *arg, struct tcp_pcb *pcb, struct pb uint64_t total = 0, idle = 0; if (!strcmp(microkit_name, "client0")) { - total = __atomic_load_n(&bench->ts, __ATOMIC_RELAXED) - start; - idle = __atomic_load_n(&bench->ccount, __ATOMIC_RELAXED) - idle_ccount_start; + for (int i = 0; i < MAX_CORES; i++) { + if (active_cores & (1 << i)) { + struct bench *core_ccounts = (struct bench *)(idle_ccounts_vaddr + IDLE_CCOUNTS_SIZE * i); + total += __atomic_load_n(&core_ccounts->ts, __ATOMIC_RELAXED) - core_ccount_start[i]; + idle += __atomic_load_n(&core_ccounts->ccount, __ATOMIC_RELAXED) - idle_ccount_start[i]; + } + } } char tbuf[21]; @@ -199,7 +210,8 @@ static err_t utilization_accept_callback(void *arg, struct tcp_pcb *newpcb, err_ int setup_utilization_socket(void) { - bench = (void *)cyclecounters_vaddr; + active_cores = bench_active_cores(); + utiliz_socket = tcp_new_ip_type(IPADDR_TYPE_V4); if (utiliz_socket == NULL) { sddf_dprintf("Failed to open a socket for listening!\n"); @@ -220,4 +232,4 @@ int setup_utilization_socket(void) tcp_accept(utiliz_socket, utilization_accept_callback); return 0; -} \ No newline at end of file +}