Skip to content

Commit

Permalink
Upstream sddf blk to use offsets, and initialise blk system with blk_…
Browse files Browse the repository at this point in the history
…config.h

Signed-off-by: Eric Chan <[email protected]>
  • Loading branch information
erichchan999 authored and Ivan-Velickovic committed Jul 5, 2024
1 parent 34ab6fa commit 3c59b74
Show file tree
Hide file tree
Showing 13 changed files with 214 additions and 110 deletions.
39 changes: 16 additions & 23 deletions examples/virtio/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,15 @@ CURRENT_DIR := $(shell pwd)

LINUX_DIR := $(VMM_TOOLS)/linux
LINUX_BLK_DIR := $(LINUX_DIR)/blk
LINUX_BLK_BOARD_DIR := $(LINUX_BLK_DIR)/board/$(BOARD)

UIO_DIR := $(LINUX_DIR)/uio
UIO_DRIVER_BLK_DIR := $(LINUX_DIR)/uio_drivers/blk

SDDF_DIR := $(abspath ../../dep/sddf)

VMM_INCLUDE := include

BLK_DRIVER_VM_DIR := board/$(BOARD)/blk_driver_vm
BLK_DRIVER_VM_DTS_DIR := $(BLK_DRIVER_VM_DIR)/dts
BLK_DRIVER_VM_ROOTFS_TMP_DIR := $(BUILD_DIR)/blk_driver_vm_rootfs
Expand Down Expand Up @@ -123,26 +128,16 @@ $(error Unsupported BOARD given)
endif
SDDF_SERIAL_DRIVER := $(SDDF_DIR)/drivers/serial/$(UART_DRIVER_DIR)

SERIAL_CONFIG_INCLUDE:= include/serial_config

ELFS := client_vmm_1.elf client_vmm_2.elf blk_driver_vmm.elf serial_virt_tx.elf serial_virt_rx.elf uart_driver.elf blk_virt.elf

BLK_DRIVER_VM_USERLEVEL_qemu_arm_virt := uio_blk_driver
BLK_DRIVER_VM_USERLEVEL_odroidc4 := uio_blk_driver
BLK_DRIVER_VM_USERLEVEL := ${BLK_DRIVER_VM_USERLEVEL_${BOARD}}
BLK_DRIVER_VM_USERLEVEL_INIT_qemu_arm_virt := qemu_blk_driver_init
BLK_DRIVER_VM_USERLEVEL_INIT_odroidc4 := blk_driver_init
BLK_DRIVER_VM_USERLEVEL_INIT := ${BLK_DRIVER_VM_USERLEVEL_INIT_${BOARD}}
BLK_DRIVER_VM_USERLEVEL := uio_blk_driver
BLK_DRIVER_VM_USERLEVEL_INIT := blk_driver_init

CLIENT_VM_1_USERLEVEL :=
CLIENT_VM_1_USERLEVEL_INIT_qemu_arm_virt := qemu_blk_client_init
CLIENT_VM_1_USERLEVEL_INIT_odroidc4 := blk_client_init
CLIENT_VM_1_USERLEVEL_INIT := ${CLIENT_VM_1_USERLEVEL_INIT_${BOARD}}
CLIENT_VM_1_USERLEVEL_INIT := blk_client_init

CLIENT_VM_2_USERLEVEL :=
CLIENT_VM_2_USERLEVEL_INIT_qemu_arm_virt := qemu_blk_client_init
CLIENT_VM_2_USERLEVEL_INIT_odroidc4 := blk_client_init
CLIENT_VM_2_USERLEVEL_INIT := ${CLIENT_VM_2_USERLEVEL_INIT_${BOARD}}
CLIENT_VM_2_USERLEVEL_INIT:= blk_client_init

IMAGE_FILE = $(BUILD_DIR)/loader.img
REPORT_FILE = $(BUILD_DIR)/report.txt
Expand Down Expand Up @@ -202,7 +197,7 @@ CFLAGS := -mstrict-align \
-Wall -Wno-unused-function -Werror \
-I$(VMM_SRC_DIR)/arch/aarch64 -I$(VMM_SRC_DIR) -I$(VMM_SRC_DIR)/util -I$(BOARD_DIR)/include \
-I$(SDDF_DIR)/include \
-I$(SERIAL_CONFIG_INCLUDE) \
-I$(VMM_INCLUDE) \
-DBOARD_$(BOARD) \
-DCONFIG_$(CONFIG) \
-target aarch64-none-elf
Expand All @@ -214,6 +209,7 @@ CFLAGS_LINUX := -g3 \
-D_GNU_SOURCE \
-I$(SDDF_DIR)/include \
-I$(LINUX_DIR)/include \
-I$(VMM_INCLUDE) \
-target aarch64-linux-gnu

LDFLAGS := -L$(BOARD_DIR)/lib
Expand Down Expand Up @@ -348,7 +344,7 @@ $(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/virtio/%.c Makefile
$(CC) -c $(CFLAGS) $< -o $@

$(BUILD_DIR)/sddf_blk_%.o: $(SDDF_BLK_COMPONENTS)/%.c Makefile
$(CC) -c $(CFLAGS) -DBLK_NUM_CLIENTS=2 $< -o $@
$(CC) -c $(CFLAGS) $< -o $@

$(BUILD_DIR)/sddf_blk_%.o: $(SDDF_BLK_UTIL)/%.c Makefile
$(CC) -c $(CFLAGS) $< -o $@
Expand All @@ -371,18 +367,15 @@ $(BUILD_DIR)/postmark: $(LINUX_BLK_DIR)/postmark.c
$(BUILD_DIR)/blk_client_init: $(LINUX_BLK_DIR)/blk_client_init
cp $< $@

$(BUILD_DIR)/blk_driver_init: $(LINUX_BLK_DIR)/blk_driver_init
cp $< $@

$(BUILD_DIR)/qemu_blk_client_init: $(LINUX_BLK_DIR)/qemu_blk_client_init
cp $< $@

$(BUILD_DIR)/qemu_blk_driver_init: $(LINUX_BLK_DIR)/qemu_blk_driver_init
$(BUILD_DIR)/blk_driver_init: $(LINUX_BLK_BOARD_DIR)/blk_driver_init
cp $< $@

$(BUILD_DIR)/%.o: $(UIO_DIR)/%.c Makefile
$(CC_USERLEVEL) -c $(CFLAGS_LINUX) $< -o $@

$(BUILD_DIR)/%.o: $(UIO_DRIVER_BLK_DIR)/%.c Makefile
$(CC_USERLEVEL) -c $(CFLAGS_LINUX) $< -o $@

$(BUILD_DIR)/uio_blk_driver: $(addprefix $(BUILD_DIR)/, $(UIO_BLK_DRIVER_OBJS))
$(CC_USERLEVEL) $(CFLAGS_LINUX) $^ -o $@

Expand Down
21 changes: 10 additions & 11 deletions examples/virtio/board/odroidc4/virtio.system
Original file line number Diff line number Diff line change
Expand Up @@ -223,28 +223,27 @@
<map mr="data_blk_driver" vaddr="0x40800000" perms="rw" cached="true" setvar_vaddr="blk_data_driver" />

<map mr="blk_config_vmm_1" vaddr="0x30000000" perms="rw" cached="false" setvar_vaddr="blk_config" />
<map mr="req_blk_vmm_1" vaddr="0x30200000" perms="rw" cached="false" setvar_vaddr="blk_req_queue" />
<map mr="resp_blk_vmm_1" vaddr="0x30400000" perms="rw" cached="false" setvar_vaddr="blk_resp_queue" />
<map mr="data_blk_vmm_1" vaddr="0x30600000" perms="rw" cached="true" setvar_vaddr="blk_data" />

<map mr="blk_config_vmm_2" vaddr="0x31000000" perms="rw" cached="false" setvar_vaddr="blk_config2" />
<map mr="req_blk_vmm_2" vaddr="0x31200000" perms="rw" cached="false" setvar_vaddr="blk_req_queue2" />
<map mr="resp_blk_vmm_2" vaddr="0x31400000" perms="rw" cached="false" setvar_vaddr="blk_resp_queue2" />
<map mr="data_blk_vmm_2" vaddr="0x31600000" perms="rw" cached="true" setvar_vaddr="blk_data2" />
<map mr="blk_config_vmm_2" vaddr="0x30200000" perms="rw" cached="false" />
<map mr="req_blk_vmm_1" vaddr="0x31000000" perms="rw" cached="false" setvar_vaddr="blk_req_queue" />
<map mr="req_blk_vmm_2" vaddr="0x31200000" perms="rw" cached="false" />
<map mr="resp_blk_vmm_1" vaddr="0x32000000" perms="rw" cached="false" setvar_vaddr="blk_resp_queue" />
<map mr="resp_blk_vmm_2" vaddr="0x32200000" perms="rw" cached="false" />
<map mr="data_blk_vmm_1" vaddr="0x33000000" perms="rw" cached="true" setvar_vaddr="blk_client_data_start"/>
<map mr="data_blk_vmm_2" vaddr="0x33200000" perms="rw" cached="true" />
</protection_domain>

<channel>
<end pd="CLIENT_VMM-1" id="3"/>
<end pd="BLK_VIRT" id="3"/>
<end pd="BLK_VIRT" id="1"/>
</channel>

<channel>
<end pd="CLIENT_VMM-2" id="3"/>
<end pd="BLK_VIRT" id="4"/>
<end pd="BLK_VIRT" id="2"/>
</channel>

<channel>
<end pd="BLK_VIRT" id="1"/>
<end pd="BLK_VIRT" id="0"/>
<end pd="BLK_DRIVER_VMM" id="3"/>
</channel>
</system>
21 changes: 10 additions & 11 deletions examples/virtio/board/qemu_arm_virt/virtio.system
Original file line number Diff line number Diff line change
Expand Up @@ -216,28 +216,27 @@
<map mr="data_blk_driver" vaddr="0x20800000" perms="rw" cached="true" setvar_vaddr="blk_data_driver" />

<map mr="blk_config_vmm_1" vaddr="0x30000000" perms="rw" cached="false" setvar_vaddr="blk_config" />
<map mr="req_blk_vmm_1" vaddr="0x30200000" perms="rw" cached="false" setvar_vaddr="blk_req_queue" />
<map mr="resp_blk_vmm_1" vaddr="0x30400000" perms="rw" cached="false" setvar_vaddr="blk_resp_queue" />
<map mr="data_blk_vmm_1" vaddr="0x30600000" perms="rw" cached="true" setvar_vaddr="blk_data" />

<map mr="blk_config_vmm_2" vaddr="0x31000000" perms="rw" cached="false" setvar_vaddr="blk_config2" />
<map mr="req_blk_vmm_2" vaddr="0x31200000" perms="rw" cached="false" setvar_vaddr="blk_req_queue2" />
<map mr="resp_blk_vmm_2" vaddr="0x31400000" perms="rw" cached="false" setvar_vaddr="blk_resp_queue2" />
<map mr="data_blk_vmm_2" vaddr="0x31600000" perms="rw" cached="true" setvar_vaddr="blk_data2" />
<map mr="blk_config_vmm_2" vaddr="0x30200000" perms="rw" cached="false" />
<map mr="req_blk_vmm_1" vaddr="0x31000000" perms="rw" cached="false" setvar_vaddr="blk_req_queue" />
<map mr="req_blk_vmm_2" vaddr="0x31200000" perms="rw" cached="false" />
<map mr="resp_blk_vmm_1" vaddr="0x32000000" perms="rw" cached="false" setvar_vaddr="blk_resp_queue" />
<map mr="resp_blk_vmm_2" vaddr="0x32200000" perms="rw" cached="false" />
<map mr="data_blk_vmm_1" vaddr="0x33000000" perms="rw" cached="true" setvar_vaddr="blk_client_data_start"/>
<map mr="data_blk_vmm_2" vaddr="0x33200000" perms="rw" cached="true" />
</protection_domain>

<channel>
<end pd="CLIENT_VMM-1" id="3"/>
<end pd="BLK_VIRT" id="3"/>
<end pd="BLK_VIRT" id="1"/>
</channel>

<channel>
<end pd="CLIENT_VMM-2" id="3"/>
<end pd="BLK_VIRT" id="4"/>
<end pd="BLK_VIRT" id="2"/>
</channel>

<channel>
<end pd="BLK_VIRT" id="1"/>
<end pd="BLK_VIRT" id="0"/>
<end pd="BLK_DRIVER_VMM" id="3"/>
</channel>
</system>
3 changes: 2 additions & 1 deletion examples/virtio/client_vmm.c
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <sddf/serial/queue.h>
#include <serial_config.h>
#include <sddf/blk/queue.h>
#include <blk_config.h>

#define GUEST_RAM_SIZE 0x6000000

Expand Down Expand Up @@ -130,7 +131,7 @@ void init(void)
blk_queue_init(&blk_queue_h,
(blk_req_queue_t *)blk_req_queue,
(blk_resp_queue_t *)blk_resp_queue,
BLK_QUEUE_SIZE);
blk_cli_queue_size(microkit_name));

/* Initialise virtIO block device */
success = virtio_mmio_blk_init(&virtio_blk,
Expand Down
115 changes: 115 additions & 0 deletions examples/virtio/include/blk_config.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
#pragma once

#include <sddf/util/string.h>
#include <sddf/blk/queue.h>

#define BLK_NUM_CLIENTS 2

#define BLK_NAME_CLI0 "CLIENT_VMM-1"
#define BLK_NAME_CLI1 "CLIENT_VMM-2"

#define BLK_QUEUE_SIZE_CLI0 1024
#define BLK_QUEUE_SIZE_CLI1 1024
#define BLK_QUEUE_SIZE_DRIV (BLK_QUEUE_SIZE_CLI0 + BLK_QUEUE_SIZE_CLI1)

#define BLK_REGION_SIZE 0x200000
#define BLK_CONFIG_REGION_SIZE_CLI0 BLK_REGION_SIZE
#define BLK_CONFIG_REGION_SIZE_CLI1 BLK_REGION_SIZE

#define BLK_DATA_REGION_SIZE_CLI0 BLK_REGION_SIZE
#define BLK_DATA_REGION_SIZE_CLI1 BLK_REGION_SIZE
#define BLK_DATA_REGION_SIZE_DRIV BLK_REGION_SIZE

#define BLK_QUEUE_REGION_SIZE_CLI0 BLK_REGION_SIZE
#define BLK_QUEUE_REGION_SIZE_CLI1 BLK_REGION_SIZE
#define BLK_QUEUE_REGION_SIZE_DRIV BLK_REGION_SIZE

_Static_assert(BLK_DATA_REGION_SIZE_CLI0 >= BLK_TRANSFER_SIZE && BLK_DATA_REGION_SIZE_CLI0 % BLK_TRANSFER_SIZE == 0,
"Client0 data region size must be a multiple of the transfer size");
_Static_assert(BLK_DATA_REGION_SIZE_CLI1 >= BLK_TRANSFER_SIZE && BLK_DATA_REGION_SIZE_CLI1 % BLK_TRANSFER_SIZE == 0,
"Client1 data region size must be a multiple of the transfer size");
_Static_assert(BLK_DATA_REGION_SIZE_DRIV >= BLK_TRANSFER_SIZE && BLK_DATA_REGION_SIZE_DRIV % BLK_TRANSFER_SIZE == 0,
"Driver data region size must be a multiple of the transfer size");

static inline blk_storage_info_t *blk_virt_cli_config_info(blk_storage_info_t *info, unsigned int id)
{
switch (id) {
case 0:
return info;
case 1:
return (blk_storage_info_t *)((uintptr_t)info + BLK_CONFIG_REGION_SIZE_CLI0);
default:
return NULL;
}
}

static inline uintptr_t blk_virt_cli_data_region(uintptr_t data, unsigned int id)
{
switch (id) {
case 0:
return data;
case 1:
return (uintptr_t)data + BLK_DATA_REGION_SIZE_CLI0;
default:
return 0;
}
}

static inline uint64_t blk_virt_cli_data_region_size(unsigned int id)
{
switch (id) {
case 0:
return BLK_DATA_REGION_SIZE_CLI0;
case 1:
return BLK_DATA_REGION_SIZE_CLI1;
default:
return 0;
}
}

static inline blk_req_queue_t *blk_virt_cli_req_queue(blk_req_queue_t *req, unsigned int id)
{
switch (id) {
case 0:
return req;
case 1:
return (blk_req_queue_t *)((uintptr_t)req + BLK_QUEUE_REGION_SIZE_CLI0);
default:
return NULL;
}
}

static inline blk_resp_queue_t *blk_virt_cli_resp_queue(blk_resp_queue_t *resp, unsigned int id)
{
switch (id) {
case 0:
return resp;
case 1:
return (blk_resp_queue_t *)((uintptr_t)resp + BLK_QUEUE_REGION_SIZE_CLI0);
default:
return NULL;
}
}

static inline uint32_t blk_virt_cli_queue_size(unsigned int id)
{
switch (id) {
case 0:
return BLK_QUEUE_SIZE_CLI0;
case 1:
return BLK_QUEUE_SIZE_CLI1;
default:
return 0;
}
}

static inline uint32_t blk_cli_queue_size(char *pd_name)
{
if (!sddf_strcmp(pd_name, BLK_NAME_CLI0)) {
return BLK_QUEUE_SIZE_CLI0;
} else if (!sddf_strcmp(pd_name, BLK_NAME_CLI1)) {
return BLK_QUEUE_SIZE_CLI1;
} else {
return 0;
}
}
File renamed without changes.
17 changes: 12 additions & 5 deletions src/virtio/block.c
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#include <microkit.h>
#include <stdint.h>
#include <stdbool.h>
#include "sddf/blk/queue.h"
#include <sddf/blk/queue.h>
#include <sddf/util/fsmalloc.h>
#include <sddf/util/ialloc.h>
#include "virtio/config.h"
#include "virtio/virtq.h"
#include "virtio/mmio.h"
Expand Down Expand Up @@ -244,7 +246,8 @@ static int virtio_blk_mmio_queue_notify(struct virtio_device *dev)
virtio_data, virtio_data_size, 0
};

err = blk_enqueue_req(&state->queue_h, READ_BLOCKS, sddf_data, sddf_block_number, sddf_count, req_id);
uintptr_t offset = sddf_data - ((struct virtio_blk_device *)dev->device_data)->data_region;
err = blk_enqueue_req(&state->queue_h, READ_BLOCKS, offset, sddf_block_number, sddf_count, req_id);
assert(!err);
break;
}
Expand Down Expand Up @@ -289,7 +292,8 @@ static int virtio_blk_mmio_queue_notify(struct virtio_device *dev)
virtio_data, virtio_data_size, aligned
};

err = blk_enqueue_req(&state->queue_h, READ_BLOCKS, sddf_data, sddf_block_number, sddf_count, req_id);
uintptr_t offset = sddf_data - ((struct virtio_blk_device *)dev->device_data)->data_region;
err = blk_enqueue_req(&state->queue_h, READ_BLOCKS, offset, sddf_block_number, sddf_count, req_id);
assert(!err);
} else {
if (!sddf_make_req_check(state, sddf_count)) {
Expand All @@ -316,7 +320,9 @@ static int virtio_blk_mmio_queue_notify(struct virtio_device *dev)

/* Copy data from virtio buffer to data buffer, create sddf write request and initialise it with data buffer */
memcpy((void *)sddf_data, (void *)virtq->desc[curr_desc_head].addr, virtq->desc[curr_desc_head].len);
err = blk_enqueue_req(&state->queue_h, WRITE_BLOCKS, sddf_data, sddf_block_number, sddf_count, req_id);

uintptr_t offset = sddf_data - ((struct virtio_blk_device *)dev->device_data)->data_region;
err = blk_enqueue_req(&state->queue_h, WRITE_BLOCKS, offset, sddf_block_number, sddf_count, req_id);
assert(!err);
}
break;
Expand Down Expand Up @@ -424,7 +430,7 @@ bool virtio_blk_handle_resp(struct virtio_blk_device *state)

err = blk_enqueue_req(&state->queue_h,
WRITE_BLOCKS,
data->sddf_data,
data->sddf_data - state->data_region,
data->sddf_block_number,
data->sddf_count,
new_sddf_id);
Expand Down Expand Up @@ -518,6 +524,7 @@ bool virtio_mmio_blk_init(struct virtio_blk_device *blk_dev,

blk_dev->storage_info = storage_info;
blk_dev->queue_h = *queue_h;
blk_dev->data_region = data_region;
blk_dev->server_ch = server_ch;

size_t sddf_data_buffers = data_region_size / BLK_TRANSFER_SIZE;
Expand Down
3 changes: 2 additions & 1 deletion src/virtio/block.h
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ struct virtio_blk_outhdr {
#define SDDF_BLK_NUM_HANDLES 1
#define SDDF_BLK_DEFAULT_HANDLE 0
/* Maximum number of buffers in sddf data region */
#define SDDF_MAX_DATA_BUFFERS 4096
#define SDDF_MAX_DATA_BUFFERS 8192

#define VIRTIO_BLK_NUM_VIRTQ 1
#define VIRTIO_BLK_DEFAULT_VIRTQ 0
Expand Down Expand Up @@ -173,6 +173,7 @@ struct virtio_blk_device {

blk_storage_info_t *storage_info;
blk_queue_handle_t queue_h;
uintptr_t data_region;
int server_ch;
};

Expand Down
File renamed without changes.
File renamed without changes.
3 changes: 0 additions & 3 deletions tools/linux/blk/qemu_blk_client_init

This file was deleted.

Loading

0 comments on commit 3c59b74

Please sign in to comment.