diff --git a/examples/virtio/blk_driver_vmm.c b/examples/virtio/blk_driver_vmm.c index 3131206e6..e0be21f0f 100644 --- a/examples/virtio/blk_driver_vmm.c +++ b/examples/virtio/blk_driver_vmm.c @@ -73,6 +73,13 @@ static void register_passthrough_irq(int irq, microkit_channel irq_ch) { #define UIO_BLK_IRQ 50 #define VSWITCH_BLK 3 +uintptr_t uio_irq; + +void uio_ack(size_t vcpu_id, int irq, void *cookie) { + LOG_VMM("uio_ack!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); + microkit_irq_ack(VSWITCH_BLK); +} + /* Virtio Console */ #define SERIAL_MUX_TX_CH 1 #define SERIAL_MUX_RX_CH 2 @@ -169,7 +176,7 @@ void init(void) { assert(success); /* Register UIO irq */ - virq_register(GUEST_VCPU_ID, UIO_BLK_IRQ, &dummy_ack, NULL); + virq_register(GUEST_VCPU_ID, UIO_BLK_IRQ, &uio_ack, NULL); /* Finally start the guest */ guest_start(GUEST_VCPU_ID, kernel_pc, GUEST_DTB_VADDR, GUEST_INIT_RAM_DISK_VADDR); diff --git a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.c b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.c index 06f7861f0..d88b4cae6 100644 --- a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.c +++ b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.c @@ -15,77 +15,74 @@ #include "libuio.h" #include "block.h" +/* Uncomment this to enable debug logging */ +#define DEBUG_UIO_BLOCK + +#if defined(DEBUG_UIO_BLOCK) +#define LOG_UIO_BLOCK(...) do{ printf("UIO_DRIVER(BLOCK): "); printf(__VA_ARGS__); }while(0) +#else +#define LOG_UIO_BLOCK(...) do{}while(0) +#endif + +#define LOG_UIO_BLOCK_ERR(...) do{ printf("UIO_DRIVER(BLOCK)|ERROR: "); printf(__VA_ARGS__); }while(0) + #define BLK_CH 3 blk_queue_handle_t h; uintptr_t blk_data; blk_storage_info_t *blk_config; -/* as much as we don't want to expose the UIO interface to the guest driver, - * over engineering is still not a good idea. For not I think the guest driver - * should be aware of the Linux UIO interface - */ -void init(int fd) +int driver_init(void **maps, int num_maps) { - blk_req_queue_t *req_queue; - if ((req_queue = mmap(NULL, 0x200000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)) == (void *) -1) { - printf("req_queue: mmap failed, errno: %d\n", errno); - close(fd); - exit(1); - } - - blk_resp_queue_t *resp_queue; - if ((resp_queue = mmap(NULL, 0x200000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 1 * getpagesize())) == (void *) -1) { - printf("resp_queue: mmap failed, errno: %d\n", errno); - close(fd); - exit(2); - } - - void *data; - if ((data = mmap(NULL, 0x200000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 2 * getpagesize())) == (void *) -1) { - printf("blk_data: mmap failed, errno: %d\n", errno); - close(fd); - exit(3); - } - blk_data = (uintptr_t)data; - - if ((blk_config = mmap(NULL, 0x200000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 3 * getpagesize())) == (void *) -1) { - printf("blk_config: mmap failed, errno: %d\n", errno); - close(fd); - exit(4); + if (num_maps != 4) { + LOG_UIO_BLOCK_ERR("Expecting 4 maps, got %d\n", num_maps); + return -1; } + + blk_req_queue_t *req_queue = (blk_req_queue_t *)maps[0]; + blk_resp_queue_t *resp_queue = (blk_resp_queue_t *)maps[1]; + blk_data = (uintptr_t)maps[2]; + blk_config = (blk_storage_info_t *)maps[3]; blk_queue_init(&h, req_queue, resp_queue, false, BLK_REQ_QUEUE_SIZE, BLK_RESP_QUEUE_SIZE); + + // @TODO, @ericc: Query the block device we have and fill in the blk_config + // just random numbers I've chosen for now + blk_config->blocksize = 1024; + blk_config->size = 1024 * 1024 * 1024; + blk_config->read_only = false; + blk_config->ready = true; + + // Notify the other side that driver is ready + uio_notify(); + + return 0; } -void notified(microkit_channel ch) +void driver_notified() { - if (ch == BLK_CH) { - printf("Received notification on channel %d\n", ch); + blk_request_code_t ret_code; + uintptr_t ret_addr; + uint32_t ret_sector; + uint16_t ret_count; + uint32_t ret_id; - blk_request_code_t ret_code; - uintptr_t ret_addr; - uint32_t ret_sector; - uint16_t ret_count; - uint32_t ret_id; + while (!blk_req_queue_empty(&h)) { + blk_dequeue_req(&h, &ret_code, &ret_addr, &ret_sector, &ret_count, &ret_id); + printf("Received command: code=%d, addr=%p, sector=%d, count=%d, id=%d\n", ret_code, (void *)ret_addr, ret_sector, ret_count, ret_id); - while (!blk_req_queue_empty(&h)) { - blk_dequeue_req(&h, &ret_code, &ret_addr, &ret_sector, &ret_count, &ret_id); - printf("Received command: code=%d, addr=%p, sector=%d, count=%d, id=%d\n", ret_code, (void *)ret_addr, ret_sector, ret_count, ret_id); - - blk_response_status_t status = SUCCESS; - uintptr_t addr = ret_addr; - uint16_t count = ret_count; - uint16_t success_count = ret_count; - uint32_t id = ret_id; - //@ericc: what happens if this response is dropped? should we have a timeout in client? - if (blk_resp_queue_full(&h)) { - printf("Response ring is full, dropping response\n"); - continue; - } - blk_enqueue_resp(&h, status, addr, count, success_count, id); + blk_response_status_t status = SUCCESS; + uintptr_t addr = ret_addr; + uint16_t count = ret_count; + uint16_t success_count = ret_count; + uint32_t id = ret_id; + //@ericc: what happens if this response is dropped? should we have a timeout in client? + if (blk_resp_queue_full(&h)) { + printf("Response ring is full, dropping response\n"); + continue; } - - microkit_notify(BLK_CH); + blk_enqueue_resp(&h, status, addr, count, success_count, id); } + + uio_notify(); } diff --git a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.h b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.h index 77509cba5..b5361f1a4 100644 --- a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.h +++ b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/block.h @@ -1,9 +1,9 @@ /* - * Copyright 2023, UNSW + * Copyright 2024, UNSW * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once -void init(int fd); -void notified(microkit_channel ch); +int driver_init(void **maps, int num_maps); +void driver_notified(); diff --git a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.c b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.c index f4edc948e..6273e9a73 100644 --- a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.c +++ b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.c @@ -1,5 +1,5 @@ /* - * Copyright 2023, UNSW + * Copyright 2024, UNSW * * SPDX-License-Identifier: BSD-2-Clause */ @@ -13,70 +13,180 @@ #include #include #include +#include +#include +#include #include #include #include -#include +#include #include "libuio.h" #include "block.h" -/* - * the kernel UIO driver we use (uio_pdrv_genirq) only handles one IRQ source. In the case that - * we have multiple channels, we have to store the channel id somewhere in the shared memorys - */ -typedef struct connection_info { - int32_t irq; /* we need this to notify */ - microkit_channel notify_ch; /* the channel number to notify */ - microkit_channel notified_ch; /* the channel number to be notified */ -} connection_info_t; +/* Uncomment this to enable debug logging */ +#define DEBUG_UIO + +#if defined(DEBUG_UIO) +#define LOG_UIO(...) do{ printf("UIO_DRIVER: "); printf(__VA_ARGS__); }while(0) +#else +#define LOG_UIO(...) do{}while(0) +#endif + +#define LOG_UIO_ERR(...) do{ printf("UIO_DRIVER|ERROR: "); printf(__VA_ARGS__); }while(0) + + +#define UIO_DEV_NAME "/dev/uio0" +#define UIO_MAPS_DIR "/sys/class/uio/uio0/maps" +#define UIO_MAPS_MAX_NAME 64 +#define UIO_MAX_MAPS 32 static struct pollfd pfd; -static connection_info_t *conn; +static void *maps[UIO_MAX_MAPS]; +static int num_maps; /* * Just happily abort if the user can't be bother to provide these functions */ -__attribute__((weak)) void init(int fd) +__attribute__((weak)) int driver_init(void **maps, int num_maps) { assert(!"should not be here!"); } -__attribute__((weak)) void notified(microkit_channel ch) +__attribute__((weak)) void driver_notified() { assert(!"should not be here!"); } -void microkit_notify(microkit_channel ch) +void uio_notify() { - conn->notify_ch = ch; - - // this *should* re-enable the IRQ according to the UIO spec. Not sure why it doesn't - // conn->sirq = 1; - - // writing directly to the file also re-enables the IRQ + // writing 1 to the uio device re-enables/acks the IRQ int32_t one = 1; int ret = write(pfd.fd, &one, 4); if (ret < 0) { - printf("ret val: %d, errno: %d\n", ret, errno); + LOG_UIO_ERR("writing 1 to device failed with ret val: %d, errno: %d\n", ret, errno); } fsync(pfd.fd); } -int main(int argc, char *argv[]) { - if (argc != 3) { - printf("Usage: %s <#mapping>\n\n" - "TODO(@jade): say something useful here\n", - argv[0]); - return 1; +static int uio_num_maps() { + const char *path = UIO_MAPS_DIR; + DIR *dir; + struct dirent *entry; + struct stat statbuf; + regex_t regex; + int count = 0; + + // Compile regex + if (regcomp(®ex, "^map[0-9]+$", REG_EXTENDED) != 0) { + LOG_UIO_ERR("Could not compile regex\n"); + return -1; } - char *uio_device_name = argv[1]; - int num_mapping = atoi(argv[2]); - assert(num_mapping > 0); + // Open directory + dir = opendir(path); + if (dir == NULL) { + LOG_UIO_ERR("Failed to open directory\n"); + return -1; + } + + // Read directory entries + while ((entry = readdir(dir)) != NULL) { + char fullPath[UIO_MAPS_MAX_NAME]; + + int len = snprintf(fullPath, sizeof(fullPath), "%s/%s", path, entry->d_name); + if (len < 0 || len >= sizeof(fullPath)) { + LOG_UIO_ERR("Failed to create full path\n"); + return -1; + }; + + // Check if entry is a directory + if (stat(fullPath, &statbuf) == 0 && S_ISDIR(statbuf.st_mode)) { + // skip over . and .. + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + + // Check if directory name matches regex + if (regexec(®ex, entry->d_name, 0, NULL, 0) == 0) { + count++; + LOG_UIO("Match found: %s\n", entry->d_name); + } + } + } - printf("UIO device name: %s, number of mappings: %d\n", uio_device_name, num_mapping); + LOG_UIO("Total directories matching 'map[0-9]+': %d\n", count); + + // Cleanup + closedir(dir); + regfree(®ex); + + return count; +} + +static size_t uio_map_size(int map_num) { + char path[UIO_MAPS_MAX_NAME]; + char buf[UIO_MAPS_MAX_NAME]; + + int len = snprintf(path, sizeof(path), "%s/map%d/size", UIO_MAPS_DIR, map_num); + if (len < 0 || len >= sizeof(path)) { + LOG_UIO_ERR("Failed to create map%d size path string\n", map_num); + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) { + LOG_UIO_ERR("Failed to open %s\n", path); + return -1; + } + ssize_t ret = read(fd, buf, sizeof(buf)); + if (ret < 0) { + LOG_UIO_ERR("Failed to read map%d size\n", map_num); + return -1; + } + close(fd); + + return strtoul(buf, NULL, 0); +} + +static int uio_map_init(int fd) +{ + num_maps = uio_num_maps(); + if (num_maps < 0) { + LOG_UIO_ERR("Failed to get number of maps\n"); + return -1; + } + if (num_maps == 0) { + LOG_UIO_ERR("No maps found\n"); + return -1; + } + if (num_maps > UIO_MAX_MAPS) { + LOG_UIO_ERR("too many maps, num_maps: %d, maximum is 16\n", num_maps); + close(fd); + return -1; + } + + for (int i=0; i= 0); - printf("receives irq, count: %d\n", irq_count); + LOG_UIO("received irq, count: %d\n", irq_count); /* clear the return event(s). */ pfd.revents = 0; /* wake the guest driver up to do some real works */ - notified(conn->notified_ch); + driver_notified(); } return 0; } + diff --git a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.h b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.h index 5bb455c12..1ed58d6e5 100644 --- a/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.h +++ b/examples/virtio/board/qemu_arm_virt/blk_driver_vm/userlevel/libuio.h @@ -1,10 +1,9 @@ /* - * Copyright 2023, UNSW + * Copyright 2024, UNSW * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once -typedef unsigned int microkit_channel; - -void microkit_notify(microkit_channel ch); +/* Notify the VMM */ +void uio_notify(); diff --git a/examples/virtio/board/qemu_arm_virt/virtio.system b/examples/virtio/board/qemu_arm_virt/virtio.system index 69710f2b9..fb2de7228 100644 --- a/examples/virtio/board/qemu_arm_virt/virtio.system +++ b/examples/virtio/board/qemu_arm_virt/virtio.system @@ -47,6 +47,8 @@ + + @@ -68,6 +70,7 @@ + @@ -78,6 +81,8 @@ + + @@ -198,6 +203,8 @@ --> + + diff --git a/examples/virtio/client_vmm_1.c b/examples/virtio/client_vmm_1.c index ff0d5c303..919a7c7d0 100644 --- a/examples/virtio/client_vmm_1.c +++ b/examples/virtio/client_vmm_1.c @@ -105,11 +105,11 @@ static struct virtio_device virtio_console; uintptr_t blk_req_queue; uintptr_t blk_resp_queue; uintptr_t blk_data; +uintptr_t blk_config; blk_queue_handle_t blk_queue_handle; static blk_queue_handle_t *blk_queue_handles[SDDF_BLK_NUM_HANDLES]; size_t blk_ch[SDDF_BLK_NUM_CH]; -blk_storage_info_t blk_config; blk_data_region_t blk_data_region; bitarray_t blk_data_region_avail_bitarr; @@ -210,10 +210,6 @@ void init(void) { blk_data_region_handlers[SDDF_BLK_DEFAULT_HANDLE] = &blk_data_region; /* Initialise channel */ blk_ch[SDDF_BLK_DEFAULT_CH_INDEX] = BLK_CH; - // @ericc: Grab these values from driver in the future, for now hard code here - blk_config.blocksize = 1024; - blk_config.size = 10000; - blk_config.ready = true; /* Initialise virtIO block device */ success = virtio_mmio_device_init(&virtio_blk, BLK, VIRTIO_BLK_BASE, VIRTIO_BLK_SIZE, VIRTIO_BLK_IRQ, &blk_config, (void **)blk_data_region_handlers, (void **)blk_queue_handles, blk_ch); diff --git a/src/virtio/block.c b/src/virtio/block.c index 5f45f46b5..840345672 100644 --- a/src/virtio/block.c +++ b/src/virtio/block.c @@ -37,7 +37,7 @@ static void virtio_blk_mmio_reset(struct virtio_device *dev) { // Poll ((blk_storage_info_t *)dev->config)->ready until it is ready while (!((blk_storage_info_t *)dev->config)->ready) { - LOG_BLOCK("waiting for device to be ready\n"); + // LOG_BLOCK("waiting for device to be ready\n"); } dev->vqs[VIRTIO_BLK_DEFAULT_VIRTQ].ready = 0; @@ -491,7 +491,11 @@ void virtio_blk_handle_resp(struct virtio_device *dev) { uint16_t sddf_ret_count; uint16_t sddf_ret_success_count; uint32_t sddf_ret_id; + + // @ericc: we need to know if we handled any responses so we can inject an interrupt + bool handled; while (!blk_resp_queue_empty(queue_handle)) { + handled = true; blk_dequeue_resp(queue_handle, &sddf_ret_status, &sddf_ret_addr, &sddf_ret_count, &sddf_ret_success_count, &sddf_ret_id); /* Freeing and retrieving request store */ @@ -531,7 +535,9 @@ void virtio_blk_handle_resp(struct virtio_device *dev) { virtio_blk_used_buffer(dev, virtio_desc); } - virtio_blk_used_buffer_virq_inject(dev); + if (handled) { + virtio_blk_used_buffer_virq_inject(dev); + } } void virtio_blk_handle_config_change(struct virtio_device *dev) {