From d01802b31f96fd3e1120443f9b3035452596951a Mon Sep 17 00:00:00 2001 From: Ivan Velickovic Date: Fri, 13 Sep 2024 15:14:44 +1000 Subject: [PATCH 01/60] init --- build.zig | 33 ++++++ drivers/pcie/starfive/pcie.c | 31 ++++++ examples/pcie/board/star64/pcie.system | 15 +++ examples/pcie/build.zig | 144 +++++++++++++++++++++++++ examples/pcie/build.zig.zon | 15 +++ examples/pcie/client.c | 37 +++++++ 6 files changed, 275 insertions(+) create mode 100644 drivers/pcie/starfive/pcie.c create mode 100644 examples/pcie/board/star64/pcie.system create mode 100644 examples/pcie/build.zig create mode 100644 examples/pcie/build.zig.zon create mode 100644 examples/pcie/client.c diff --git a/build.zig b/build.zig index 8a391a5da..b4907cf03 100644 --- a/build.zig +++ b/build.zig @@ -42,6 +42,10 @@ const DriverClass = struct { ds3231, pn532, }; + + const Pcie = enum { + starfive, + }; }; const util_src = [_][]const u8{ @@ -244,6 +248,29 @@ fn addNetworkDriver( }); driver.addIncludePath(net_config_include); driver.addIncludePath(b.path(b.fmt("drivers/network/{s}/", .{ @tagName(class) }))); + driver.linkLibrary(util); + + return driver; +} + +fn addPcieDriver( + b: *std.Build, + util: *std.Build.Step.Compile, + class: DriverClass.Pcie, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, +) *std.Build.Step.Compile { + const driver = addPd(b, .{ + .name = b.fmt("driver_pcie_{s}.elf", .{ @tagName(class) }), + .target = target, + .optimize = optimize, + .strip = false, + }); + const source = b.fmt("drivers/pcie/{s}/pcie.c", .{ @tagName(class) }); + driver.addCSourceFile(.{ + .file = b.path(source), + }); + driver.addIncludePath(b.path(b.fmt("drivers/pcie/{s}/", .{ @tagName(class) }))); driver.addIncludePath(b.path("include")); driver.linkLibrary(util); @@ -479,4 +506,10 @@ pub fn build(b: *std.Build) void { net_copy.linkLibrary(util); net_copy.linkLibrary(util_putchar_debug); b.installArtifact(net_copy); + + inline for (std.meta.fields(DriverClass.Pcie)) |class| { + const driver = addPcieDriver(b, util, @enumFromInt(class.value), target, optimize); + driver.linkLibrary(util_putchar_debug); + b.installArtifact(driver); + } } diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c new file mode 100644 index 000000000..d2d2e9a54 --- /dev/null +++ b/drivers/pcie/starfive/pcie.c @@ -0,0 +1,31 @@ +/* + * Copyright 2024, UNSW + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +uint32_t *pcie_regs; +uint32_t *pcie_config; + +void init() +{ + sddf_dprintf("pcie driver starting!\n"); + + sddf_dprintf("PRINTING PCIE REGS\n"); + for (int i = 0; i < 1024; i++) { + sddf_dprintf("[%04d]: 0x%x\n", i, pcie_regs[i]); + } + sddf_dprintf("PRINTING PCIE CONFIG\n"); + for (int i = 0; i < 1024; i++) { + sddf_dprintf("[%04d]: 0x%x\n", i, pcie_config[i]); + } +} + +void notified(microkit_channel ch) +{ +} + diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system new file mode 100644 index 000000000..b8ad252b1 --- /dev/null +++ b/examples/pcie/board/star64/pcie.system @@ -0,0 +1,15 @@ + + + + + + + + + + + diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig new file mode 100644 index 000000000..81bee24a8 --- /dev/null +++ b/examples/pcie/build.zig @@ -0,0 +1,144 @@ +// +// Copyright 2024, UNSW +// SPDX-License-Identifier: BSD-2-Clause +// +const std = @import("std"); + +const MicrokitBoard = enum { + star64, +}; + +const Target = struct { + board: MicrokitBoard, + zig_target: std.Target.Query, +}; + +const targets = [_]Target{ + .{ + .board = MicrokitBoard.star64, + .zig_target = std.Target.Query{ + .cpu_arch = .riscv64, + .cpu_model = .{ .explicit = &std.Target.riscv.cpu.baseline_rv64 }, + .os_tag = .freestanding, + .abi = .none, + }, + }, +}; + +fn findTarget(board: MicrokitBoard) std.Target.Query { + for (targets) |target| { + if (board == target.board) { + return target.zig_target; + } + } + + std.log.err("Board '{}' is not supported\n", .{board}); + std.posix.exit(1); +} + +const ConfigOptions = enum { debug, release, benchmark }; + +pub fn build(b: *std.Build) void { + const optimize = b.standardOptimizeOption(.{}); + + // Getting the path to the Microkit SDK before doing anything else + const microkit_sdk_arg = b.option([]const u8, "sdk", "Path to Microkit SDK"); + if (microkit_sdk_arg == null) { + std.log.err("Missing -Dsdk=/path/to/sdk argument being passed\n", .{}); + std.posix.exit(1); + } + const microkit_sdk = microkit_sdk_arg.?; + + const microkit_config_option = b.option(ConfigOptions, "config", "Microkit config to build for") orelse ConfigOptions.debug; + const microkit_config = @tagName(microkit_config_option); + + // Get the Microkit SDK board we want to target + const microkit_board_option = b.option(MicrokitBoard, "board", "Microkit board to target"); + + if (microkit_board_option == null) { + std.log.err("Missing -Dboard= argument being passed\n", .{}); + std.posix.exit(1); + } + const target = b.resolveTargetQuery(findTarget(microkit_board_option.?)); + const microkit_board = @tagName(microkit_board_option.?); + + const microkit_board_dir = b.fmt("{s}/board/{s}/{s}", .{ microkit_sdk, microkit_board, microkit_config }); + const microkit_tool = b.fmt("{s}/bin/microkit", .{microkit_sdk}); + const libmicrokit = b.fmt("{s}/lib/libmicrokit.a", .{microkit_board_dir}); + const libmicrokit_include = b.fmt("{s}/include", .{microkit_board_dir}); + const libmicrokit_linker_script = b.fmt("{s}/lib/microkit.ld", .{microkit_board_dir}); + + const sddf_dep = b.dependency("sddf", .{ + .target = target, + .optimize = optimize, + .libmicrokit = @as([]const u8, libmicrokit), + .libmicrokit_include = @as([]const u8, libmicrokit_include), + .libmicrokit_linker_script = @as([]const u8, libmicrokit_linker_script), + }); + + const driver_class = switch (microkit_board_option.?) { + .star64 => "starfive", + }; + + const driver = sddf_dep.artifact(b.fmt("driver_pcie_{s}.elf", .{ driver_class })); + // This is required because the SDF file is expecting a different name to the artifact we + // are dealing with. + const driver_install = b.addInstallArtifact(driver, .{ .dest_sub_path = "pcie_driver.elf" }); + + const client = b.addExecutable(.{ + .name = "client.elf", + .target = target, + .optimize = optimize, + .strip = false, + }); + + client.addCSourceFile(.{ .file = b.path("client.c") }); + client.addIncludePath(sddf_dep.path("include")); + client.linkLibrary(sddf_dep.artifact("util")); + client.linkLibrary(sddf_dep.artifact("util_putchar_debug")); + + client.addIncludePath(.{ .cwd_relative = libmicrokit_include }); + client.addObjectFile(.{ .cwd_relative = libmicrokit }); + client.setLinkerScriptPath(.{ .cwd_relative = libmicrokit_linker_script }); + + b.installArtifact(client); + + const system_description_path = b.fmt("board/{s}/pcie.system", .{microkit_board}); + const final_image_dest = b.getInstallPath(.bin, "./loader.img"); + const microkit_tool_cmd = b.addSystemCommand(&[_][]const u8{ + microkit_tool, + system_description_path, + "--search-path", b.getInstallPath(.bin, ""), + "--board", microkit_board, + "--config", microkit_config, + "-o", final_image_dest, + "-r", b.getInstallPath(.prefix, "./report.txt") + }); + microkit_tool_cmd.step.dependOn(b.getInstallStep()); + microkit_tool_cmd.step.dependOn(&driver_install.step); + const microkit_step = b.step("microkit", "Compile and build the final bootable image"); + microkit_step.dependOn(µkit_tool_cmd.step); + b.default_step = microkit_step; + + const loader_arg = b.fmt("loader,file={s},addr=0x70000000,cpu-num=0", .{final_image_dest}); + if (std.mem.eql(u8, microkit_board, "qemu_virt_aarch64")) { + const qemu_cmd = b.addSystemCommand(&[_][]const u8{ + "qemu-system-aarch64", + "-machine", + "virt,virtualization=on,highmem=off,secure=off", + "-cpu", + "cortex-a53", + "-serial", + "mon:stdio", + "-device", + loader_arg, + "-m", + "2G", + "-nographic", + }); + qemu_cmd.step.dependOn(b.default_step); + const simulate_step = b.step("qemu", "Simulate the image using QEMU"); + simulate_step.dependOn(&qemu_cmd.step); + } +} + diff --git a/examples/pcie/build.zig.zon b/examples/pcie/build.zig.zon new file mode 100644 index 000000000..450929c7d --- /dev/null +++ b/examples/pcie/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = "sddf_timer_example", + .version = "1.0.0", + + .dependencies = .{ + .sddf = .{ + .path = "../../" + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + } +} + diff --git a/examples/pcie/client.c b/examples/pcie/client.c new file mode 100644 index 000000000..ebbcf4281 --- /dev/null +++ b/examples/pcie/client.c @@ -0,0 +1,37 @@ +/* + * Copyright 2024, UNSW + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +#define TIMER_CHANNEL 1 + +void notified(microkit_channel ch) +{ + /* + * In this example we only have a single possible channel, + * so we know it's a notification from the driver telling + * us that the timeout we set has gone off. + */ + sddf_printf("CLIENT|INFO: Got a timeout!\n"); + /* Get the current time */ + uint64_t time = sddf_timer_time_now(TIMER_CHANNEL); + sddf_printf("CLIENT|INFO: Now the time (in nanoseconds) is: %lu\n", time); + /* Set another timeout */ + sddf_timer_set_timeout(TIMER_CHANNEL, NS_IN_S); +} + +void init(void) +{ + // lets get the time! + uint64_t time = sddf_timer_time_now(TIMER_CHANNEL); + sddf_printf("CLIENT|INFO: The time now is: %lu\n", time); + + // lets set a timeout + sddf_printf("CLIENT|INFO: Setting a time out for 1 second\n"); + sddf_timer_set_timeout(TIMER_CHANNEL, NS_IN_S); +} From ea36e5219208e0748781c31e42ab33b95468a8c5 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 17 Sep 2024 14:20:54 +1000 Subject: [PATCH 02/60] Basic PCIE bus enumeration Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 73 ++++++++++++++++++++++---- examples/pcie/board/star64/pcie.system | 2 +- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index d2d2e9a54..88369cc3e 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -8,24 +8,77 @@ #include #include -uint32_t *pcie_regs; -uint32_t *pcie_config; +/* + References: + + [PCIe-2.0] PCI Express 2.0 Base Specification Revision 0.9 (Sep 11, 2006). + https://community.intel.com/cipcp26785/attachments/cipcp26785/fpga-intellectual-property/8220/1/PCI_Express_Base_Specification_v20.pdf +*/ + +void *pcie_regs; + +/* This is the PCI Enhanced Configuration Access Mechanism, + See [PCIe-2.0] §7.22 +*/ +uintptr_t pcie_config; +#define PCIE_CONFIG_SIZE 0x1000000 + +/* bus between [0, 256) + device between [0, 31) + function between [0, 8) +*/ +uintptr_t get_bdf_offset(uint8_t bus, uint8_t device, uint8_t function) +{ + /* [PCIe-2.0] Table 7-1 Enhanced Configuration Address Mapping */ + + uintptr_t offset = ((uint32_t)bus << 20) | ((uint32_t)device << 15) | ((uint32_t)function << 12); + + assert(offset % 4096 == 0); /* check page aligned */ + + return offset; +} + +void device_print(uint8_t bus, uint8_t device, uint8_t function) +{ + void *base = (void *)(pcie_config + get_bdf_offset(bus, device, function)); + uint16_t vendor_id = *(uint16_t *)(&base[0x0]); + if (vendor_id == 0xffff) { + return; + } + + sddf_dprintf("\nB.D.F: %02x.%02x.%01x\n", bus, device, function); + sddf_dprintf("vendor ID: 0x%04x\n", vendor_id); + sddf_dprintf("device ID: 0x%04x\n", *(uint16_t *)(&base[0x2])); + sddf_dprintf("command register: 0x%04x\n", *(uint16_t *)(&base[0x4])); + sddf_dprintf("status register: 0x%04x\n", *(uint16_t *)(&base[0x6])); + sddf_dprintf("class code | revision ID: 0x%04x\n", *(uint32_t *)(&base[0x8])); + // sddf_dprintf("cache line size: 0x%02x\n", base[0xC]); + // sddf_dprintf("master latency timer: 0x%02x\n", base[0xD]); + sddf_dprintf("header type: 0x%02x\n", *(uint8_t *)(&base[0xE])); + sddf_dprintf("BIST: 0x%02x\n", *(uint8_t *)(&base[0xF])); +} void init() { sddf_dprintf("pcie driver starting!\n"); - sddf_dprintf("PRINTING PCIE REGS\n"); - for (int i = 0; i < 1024; i++) { - sddf_dprintf("[%04d]: 0x%x\n", i, pcie_regs[i]); - } - sddf_dprintf("PRINTING PCIE CONFIG\n"); - for (int i = 0; i < 1024; i++) { - sddf_dprintf("[%04d]: 0x%x\n", i, pcie_config[i]); + for (uint8_t bus = 0; bus < 256; bus++) { + for (uint8_t device = 0; device < 32; device++) { + for (uint8_t function = 0; function < 8; function++) { + uintptr_t offset = get_bdf_offset(bus, device, function); + if (offset >= PCIE_CONFIG_SIZE) { + goto out; + } + + device_print(bus, device, function); + } + } } + +out: + sddf_dprintf("Done\n"); } void notified(microkit_channel ch) { } - diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index b8ad252b1..ab31c950d 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -6,7 +6,7 @@ --> - + From f41335b6c0d01ad817d265044c98bc243b762da9 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 17 Sep 2024 15:25:43 +1000 Subject: [PATCH 03/60] Use a pcie_header struct Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 34 +++++++++------------ drivers/pcie/starfive/pcie.h | 57 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 drivers/pcie/starfive/pcie.h diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 88369cc3e..9f331fef5 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -8,12 +8,7 @@ #include #include -/* - References: - - [PCIe-2.0] PCI Express 2.0 Base Specification Revision 0.9 (Sep 11, 2006). - https://community.intel.com/cipcp26785/attachments/cipcp26785/fpga-intellectual-property/8220/1/PCI_Express_Base_Specification_v20.pdf -*/ +#include "pcie.h" void *pcie_regs; @@ -40,22 +35,21 @@ uintptr_t get_bdf_offset(uint8_t bus, uint8_t device, uint8_t function) void device_print(uint8_t bus, uint8_t device, uint8_t function) { - void *base = (void *)(pcie_config + get_bdf_offset(bus, device, function)); - uint16_t vendor_id = *(uint16_t *)(&base[0x0]); - if (vendor_id == 0xffff) { + void *config_base = (void *)(pcie_config + get_bdf_offset(bus, device, function)); + pcie_header_t *header = (pcie_header_t *)config_base; + + if (header->vendor_id == PCIE_VENDOR_INVALID) { return; } - sddf_dprintf("\nB.D.F: %02x.%02x.%01x\n", bus, device, function); - sddf_dprintf("vendor ID: 0x%04x\n", vendor_id); - sddf_dprintf("device ID: 0x%04x\n", *(uint16_t *)(&base[0x2])); - sddf_dprintf("command register: 0x%04x\n", *(uint16_t *)(&base[0x4])); - sddf_dprintf("status register: 0x%04x\n", *(uint16_t *)(&base[0x6])); - sddf_dprintf("class code | revision ID: 0x%04x\n", *(uint32_t *)(&base[0x8])); - // sddf_dprintf("cache line size: 0x%02x\n", base[0xC]); - // sddf_dprintf("master latency timer: 0x%02x\n", base[0xD]); - sddf_dprintf("header type: 0x%02x\n", *(uint8_t *)(&base[0xE])); - sddf_dprintf("BIST: 0x%02x\n", *(uint8_t *)(&base[0xF])); + sddf_dprintf("\nB.D:F: %02x:%02x.%01x\n", bus, device, function); + sddf_dprintf("vendor ID: 0x%04x\n", header->vendor_id); + sddf_dprintf("device ID: 0x%04x\n", header->device_id); + sddf_dprintf("command register: 0x%04x\n", header->command); + sddf_dprintf("status register: 0x%04x\n", header->status); + sddf_dprintf("revision ID: 0x%02x\n", header->revision_id); + sddf_dprintf("base-class code: 0x%02x | sub-class code: 0x%02x\n", header->base_class_code, header->subclass_code); + sddf_dprintf("header type: 0x%02x\n", header->header_type); } void init() @@ -76,7 +70,7 @@ void init() } out: - sddf_dprintf("Done\n"); + sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); } void notified(microkit_channel ch) diff --git a/drivers/pcie/starfive/pcie.h b/drivers/pcie/starfive/pcie.h new file mode 100644 index 000000000..68a36a4cb --- /dev/null +++ b/drivers/pcie/starfive/pcie.h @@ -0,0 +1,57 @@ +#pragma once + +/* + References: + + [PCIe-2.0] PCI Express 2.0 Base Specification Revision 0.9 (Sep 11, 2006). + https://community.intel.com/cipcp26785/attachments/cipcp26785/fpga-intellectual-property/8220/1/PCI_Express_Base_Specification_v20.pdf + + [PCI-3.0] PCI Local Bus Specification Revision 3.0 (Feb 3, 2004) + https://lekensteyn.nl/files/docs/PCI_SPEV_V3_0.pdf +*/ + +/* [PCIe-2.0] §7.5 PCI-Compatible Configuration Registers + [PCI-3.0] §6.1 Configuration Space Organization + + All of these registers are read-only. +*/ +typedef struct pcie_header { + /* This field identifies the manufacturer of the device. + FFFFh is an invalid value for Vendor ID. */ + uint16_t vendor_id; + /* This field identifies the particular device. + This identifier is allocated by the vendor.*/ + uint16_t device_id; + /* Provides coarse control over a device's ability to generate and respond to PCI cycles. + See [PCI-3.0] §6.2.2 Device Control and [PCIe-2.0] §7.5.1.1 */ + uint16_t command; + /* The Status register is used to record status information for PCI bus related events. + See [PCI-3.0] §6.2.3 Device Status and [PCIe-2.0] §7.5.1.2 */ + uint16_t status; + /* + This register specifies a device specific revision identifier. The value + is chosen by the vendor. Zero is an acceptable value. This field + should be viewed as a vendor defined extension to the Device ID. + */ + uint8_t revision_id; + /* A specific register-level programming interface (if any). */ + uint8_t class_code_programming_interface; + /* sub-class code which identifies more specifically the function of the device*/ + uint8_t subclass_code; + /* Broadly classifies the type of function the device performs. */ + uint8_t base_class_code; + /* This field is implemented by PCI Express devices as a read-write field + for legacy compatibility purposes but has no effect on any PCI Express + device behavior. */ + uint8_t cacheline_size; + /* The Latency Timer does not apply to PCI Express. */ + uint8_t latency_timer; + /* This byte identifies the layout of the second part of the predefined header. */ + uint8_t header_type; + /* Built-in Self Test. Optional. */ + uint8_t bist; +} __attribute__((packed)) pcie_header_t; + +_Static_assert(sizeof(pcie_header_t) == 16, "PCI Common Configuration Space Header must be 16 bytes"); + +#define PCIE_VENDOR_INVALID 0xffff From 934783fe21230d60b722ea69503c379be978b47e Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 17 Sep 2024 15:26:09 +1000 Subject: [PATCH 04/60] Handle JH7110/PLDA pcie errata Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 9f331fef5..55c4afb77 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -42,6 +42,21 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) return; } + /* + See: plda_pcie_addr_valid() in U-Boot + https://lore.kernel.org/u-boot/20230423105859.125764-2-minda.chen@starfivetech.com/ + + In the secondary bus of host bridge, can only access bus device 0; + all other devices are duplicates of device 0. + + @todo there appears to be some way to change the secondary bus number? + see e.g. plda_pcie_config_write() + so it might not always be bus number 1. + */ + if (bus == 1 && device > 0) { + return; + } + sddf_dprintf("\nB.D:F: %02x:%02x.%01x\n", bus, device, function); sddf_dprintf("vendor ID: 0x%04x\n", header->vendor_id); sddf_dprintf("device ID: 0x%04x\n", header->device_id); From aedd8878983e16322222cffc25ab27fb403e1a79 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 17 Sep 2024 16:33:32 +1000 Subject: [PATCH 05/60] type 0 pcie header decoding Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 39 ++++++++++++++++++++++++++++++++++++ drivers/pcie/starfive/pcie.h | 28 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 55c4afb77..e4efc2ab0 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -65,6 +65,45 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("revision ID: 0x%02x\n", header->revision_id); sddf_dprintf("base-class code: 0x%02x | sub-class code: 0x%02x\n", header->base_class_code, header->subclass_code); sddf_dprintf("header type: 0x%02x\n", header->header_type); + + sddf_dprintf("\thas multi-functions: %s\n", header->header_type & PCIE_HEADER_TYPE_HAS_MULTI_FUNCTIONS ? "yes" : "no"); + sddf_dprintf("\tlayout variant: 0x%02x\n", header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK); + + if ((header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK) == PCIE_HEADER_TYPE_GENERAL) { + pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; + for (int i = 0; i < 6; i++) { + uint32_t bar = type0_header->base_address_registers[i]; + sddf_dprintf("BAR%01d: 0x%08x\n", i, bar); + if (bar == 0) { + sddf_dprintf("\tunimplemented\n"); + continue; + } + + /* TODO: Page 226-227, setting, and then reading with dual 64-bit ones */ + + if (bar & BIT(0)) { + sddf_dprintf("\tbase address for I/O\n"); + sddf_dprintf("\taddress: 0x%08x\n", bar & ~(BIT(0) | BIT(1))); + } else { + sddf_dprintf("\tbase address for memory\n"); + sddf_dprintf("\ttype: "); + switch ((bar & (BIT(1) | BIT(2))) >> 1) { + case 0b00: + sddf_dprintf("32-bit space\n"); + break; + + case 0b10: + sddf_dprintf("64-bit space\n"); + break; + + default: + sddf_dprintf("reserved\n"); + } + sddf_dprintf("\tprefetchable: %s\n", bar & BIT(3) ? "yes" : "no"); + sddf_dprintf("\taddress: %08x\n", bar & ~(BIT(4) - 1)); + } + } + } } void init() diff --git a/drivers/pcie/starfive/pcie.h b/drivers/pcie/starfive/pcie.h index 68a36a4cb..0599ed74b 100644 --- a/drivers/pcie/starfive/pcie.h +++ b/drivers/pcie/starfive/pcie.h @@ -55,3 +55,31 @@ typedef struct pcie_header { _Static_assert(sizeof(pcie_header_t) == 16, "PCI Common Configuration Space Header must be 16 bytes"); #define PCIE_VENDOR_INVALID 0xffff + +#define PCIE_HEADER_TYPE_HAS_MULTI_FUNCTIONS BIT(7) + +#define PCIE_HEADER_TYPE_LAYOUT_MASK (BIT(7) - 1) +#define PCIE_HEADER_TYPE_GENERAL 0x00 +#define PCIE_HEADER_TYPE_PCI_PCI_BRIDGE 0x01 + +/* [PCIe-2.0] §7.5.2 Type 0 Configuration Space Header */ +typedef struct pcie_header_type0 { + pcie_header_t common_header; + /* [PCI-3.0] 6.2.5. Base Addresses + Base Address Registers (BAR) can be 32-bit (1 slot) or 64-bit (2 slots). + */ + uint32_t base_address_registers[6]; + /* Points to the Card Information Structure (CIS) for a CardBus card. */ + uint32_t cardbus_cis_pointer; + uint16_t subsystem_vendor_id; + uint16_t subsystem_id; + uint32_t expansion_rom_base_address; + uint8_t capabilities_pointer; + uint8_t _reserved[7]; + uint8_t interrupt_line; + uint8_t interrupt_pin; + uint8_t min_gnt; + uint8_t max_lat; +} __attribute__((packed)) pcie_header_type0_t; + +_Static_assert(sizeof(pcie_header_type0_t) == 64, "Type 0 Configuration Space Header must be 64 bytes"); From 61c633fdcd557ad29de8e84f73eb2c914e187483 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 20 Sep 2024 10:41:27 +1000 Subject: [PATCH 06/60] Remove leftover crud from previous example Signed-off-by: julia --- examples/pcie/build.zig | 22 ---------------------- examples/pcie/client.c | 23 ++--------------------- 2 files changed, 2 insertions(+), 43 deletions(-) diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index 81bee24a8..5d4be4507 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -119,26 +119,4 @@ pub fn build(b: *std.Build) void { const microkit_step = b.step("microkit", "Compile and build the final bootable image"); microkit_step.dependOn(µkit_tool_cmd.step); b.default_step = microkit_step; - - const loader_arg = b.fmt("loader,file={s},addr=0x70000000,cpu-num=0", .{final_image_dest}); - if (std.mem.eql(u8, microkit_board, "qemu_virt_aarch64")) { - const qemu_cmd = b.addSystemCommand(&[_][]const u8{ - "qemu-system-aarch64", - "-machine", - "virt,virtualization=on,highmem=off,secure=off", - "-cpu", - "cortex-a53", - "-serial", - "mon:stdio", - "-device", - loader_arg, - "-m", - "2G", - "-nographic", - }); - qemu_cmd.step.dependOn(b.default_step); - const simulate_step = b.step("qemu", "Simulate the image using QEMU"); - simulate_step.dependOn(&qemu_cmd.step); - } } - diff --git a/examples/pcie/client.c b/examples/pcie/client.c index ebbcf4281..9379e6a07 100644 --- a/examples/pcie/client.c +++ b/examples/pcie/client.c @@ -5,33 +5,14 @@ #include #include -#include #include -#define TIMER_CHANNEL 1 - void notified(microkit_channel ch) { - /* - * In this example we only have a single possible channel, - * so we know it's a notification from the driver telling - * us that the timeout we set has gone off. - */ - sddf_printf("CLIENT|INFO: Got a timeout!\n"); - /* Get the current time */ - uint64_t time = sddf_timer_time_now(TIMER_CHANNEL); - sddf_printf("CLIENT|INFO: Now the time (in nanoseconds) is: %lu\n", time); - /* Set another timeout */ - sddf_timer_set_timeout(TIMER_CHANNEL, NS_IN_S); + } void init(void) { - // lets get the time! - uint64_t time = sddf_timer_time_now(TIMER_CHANNEL); - sddf_printf("CLIENT|INFO: The time now is: %lu\n", time); - - // lets set a timeout - sddf_printf("CLIENT|INFO: Setting a time out for 1 second\n"); - sddf_timer_set_timeout(TIMER_CHANNEL, NS_IN_S); + sddf_dprintf("Hello from client\n"); } From bbe57a9b7a64efe120e6e4f786af583d30218f56 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 20 Sep 2024 10:46:51 +1000 Subject: [PATCH 07/60] print 64 bit addresses if we have them Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index e4efc2ab0..b2b262726 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -73,7 +73,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; for (int i = 0; i < 6; i++) { uint32_t bar = type0_header->base_address_registers[i]; - sddf_dprintf("BAR%01d: 0x%08x\n", i, bar); + sddf_dprintf("BAR%01d raw val: %08x\n", i, bar); if (bar == 0) { sddf_dprintf("\tunimplemented\n"); continue; @@ -90,17 +90,19 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) switch ((bar & (BIT(1) | BIT(2))) >> 1) { case 0b00: sddf_dprintf("32-bit space\n"); + sddf_dprintf("\tfull address: 0x%08x\n", bar & ~(BIT(4) - 1)); break; case 0b10: sddf_dprintf("64-bit space\n"); + sddf_dprintf("\tfull address: 0x%08x_%08x\n", type0_header->base_address_registers[i + 1], bar & ~(BIT(4) - 1)); + i += 1; // skip one slot. break; default: sddf_dprintf("reserved\n"); } sddf_dprintf("\tprefetchable: %s\n", bar & BIT(3) ? "yes" : "no"); - sddf_dprintf("\taddress: %08x\n", bar & ~(BIT(4) - 1)); } } } From 41e55035ee810182a06152f6b1747d9558926417 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 20 Sep 2024 13:42:34 +1000 Subject: [PATCH 08/60] pcie print size Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 44 ++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index b2b262726..ff61f9957 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -36,7 +36,7 @@ uintptr_t get_bdf_offset(uint8_t bus, uint8_t device, uint8_t function) void device_print(uint8_t bus, uint8_t device, uint8_t function) { void *config_base = (void *)(pcie_config + get_bdf_offset(bus, device, function)); - pcie_header_t *header = (pcie_header_t *)config_base; + volatile pcie_header_t *header = (pcie_header_t *)config_base; if (header->vendor_id == PCIE_VENDOR_INVALID) { return; @@ -70,7 +70,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("\tlayout variant: 0x%02x\n", header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK); if ((header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK) == PCIE_HEADER_TYPE_GENERAL) { - pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; + volatile pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; for (int i = 0; i < 6; i++) { uint32_t bar = type0_header->base_address_registers[i]; sddf_dprintf("BAR%01d raw val: %08x\n", i, bar); @@ -79,8 +79,6 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) continue; } - /* TODO: Page 226-227, setting, and then reading with dual 64-bit ones */ - if (bar & BIT(0)) { sddf_dprintf("\tbase address for I/O\n"); sddf_dprintf("\taddress: 0x%08x\n", bar & ~(BIT(0) | BIT(1))); @@ -95,7 +93,43 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) case 0b10: sddf_dprintf("64-bit space\n"); - sddf_dprintf("\tfull address: 0x%08x_%08x\n", type0_header->base_address_registers[i + 1], bar & ~(BIT(4) - 1)); + if (i >= 5) { + sddf_dprintf("\tspecified 64-bit in the last slot, ignoring..."); + continue; + } + + uint32_t bar_upper = type0_header->base_address_registers[i + 1]; + + sddf_dprintf("\tfull address: 0x%08x_%08x\n", bar_upper, bar & ~(BIT(4) - 1)); + + /* [PCI-3.0] 6.2.5.1 Address Maps (Implementation Note) p227*/ + + // Decode (I/O or memory) of a register is disabled via the command register before sizing a Base Address register. + header->command &= ~(BIT(1)); + + // calculate size. + type0_header->base_address_registers[i] = 0xffffffff; + type0_header->base_address_registers[i + 1] = 0xffffffff; + + // read back + uint32_t size_lower = type0_header->base_address_registers[i]; + uint32_t size_upper = type0_header->base_address_registers[i + 1]; + uint64_t size_readback = ((uint64_t)size_upper << 32) | (size_lower); + // calculation can be done from the 32-bit value read by first clearing encoding information bits + // (bit 0 for I/O, bits 0-3 formemory), inverting all 32 bits (logical NOT), then incrementing by 1 + size_readback &= ~(BIT(3) | BIT(2) | BIT(1) | BIT(0)); + size_readback = ~size_readback; + size_readback += 1; + + sddf_dprintf("\tsize: 0x%lx\n", size_readback); + + // The original value in the Base Address register is restored before re-enabling + // decode in the command register of the device. + type0_header->base_address_registers[i] = bar; + type0_header->base_address_registers[i + 1] = bar_upper; + header->command |= BIT(1); + + i += 1; // skip one slot. break; From a507e0ce451267d54b9be6f0b1e85845c5d141af Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 20 Sep 2024 14:51:44 +1000 Subject: [PATCH 09/60] basic nvme regs read Signed-off-by: julia --- drivers/pcie/starfive/nvme.h | 11 +++++++++++ drivers/pcie/starfive/pcie.c | 25 ++++++++++++++++++++++++- examples/pcie/board/star64/pcie.system | 14 +++++++++++--- 3 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 drivers/pcie/starfive/nvme.h diff --git a/drivers/pcie/starfive/nvme.h b/drivers/pcie/starfive/nvme.h new file mode 100644 index 000000000..f7f6ad399 --- /dev/null +++ b/drivers/pcie/starfive/nvme.h @@ -0,0 +1,11 @@ +#pragma once + +/* + References: + + [NVMe-2.1] NVM Express Base Specification Revision 2.1 (Aug 5, 2024) + https://nvmexpress.org/wp-content/uploads/NVM-Express-Base-Specification-Revision-2.1-2024.08.05-Ratified.pdf + + [NVMEe-Transport-PCIe-1.1] NVMe over PCIe Transport Specification, Revision 1.1 (Aug 5, 2024) + https://nvmexpress.org/wp-content/uploads/NVM-Express-PCI-Express-Transport-Specification-Revision-1.1-2024.08.05-Ratified.pdf +*/ diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index ff61f9957..b73474701 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -10,7 +10,7 @@ #include "pcie.h" -void *pcie_regs; +// void *pcie_regs; /* This is the PCI Enhanced Configuration Access Mechanism, See [PCIe-2.0] §7.22 @@ -18,6 +18,8 @@ void *pcie_regs; uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x1000000 +uintptr_t nvme_regs; + /* bus between [0, 256) device between [0, 31) function between [0, 8) @@ -161,6 +163,27 @@ void init() out: sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); + + sddf_dprintf("Starting NVME config...\n"); + // [NVME-2.1] Figure 33 (§3.1.4) + volatile struct { + uint64_t cap; + uint32_t vs; + uint32_t intms; + uint32_t intmc; + uint32_t cc; + } *nvme_reg = (void*)nvme_regs; + + // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 + + // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 + + // [NVME-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) + + sddf_dprintf("CAP: %016lx\n", nvme_reg->cap); + sddf_dprintf("VS: %08x\n", nvme_reg->vs); + sddf_dprintf("CC: %08x\n", nvme_reg->cc); + } void notified(microkit_channel ch) diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index ab31c950d..05bdbe5c2 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -5,11 +5,19 @@ SPDX-License-Identifier: BSD-2-Clause --> - + + + + + - - + + + + + From dfff2689dc76c5c213f1129b8f3d70a124f18152 Mon Sep 17 00:00:00 2001 From: Ivan Velickovic Date: Mon, 23 Sep 2024 11:14:23 +1000 Subject: [PATCH 10/60] add -Werror for compiling pcie driver Signed-off-by: Ivan Velickovic --- build.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/build.zig b/build.zig index b4907cf03..b4b3784a8 100644 --- a/build.zig +++ b/build.zig @@ -269,6 +269,7 @@ fn addPcieDriver( const source = b.fmt("drivers/pcie/{s}/pcie.c", .{ @tagName(class) }); driver.addCSourceFile(.{ .file = b.path(source), + .flags = &.{ "-Werror" } }); driver.addIncludePath(b.path(b.fmt("drivers/pcie/{s}/", .{ @tagName(class) }))); driver.addIncludePath(b.path("include")); From f76886fcfedc75c9a6b54bd8bee9c6917af785f4 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 23 Sep 2024 11:15:37 +1000 Subject: [PATCH 11/60] curr Signed-off-by: julia --- drivers/pcie/starfive/nvme.h | 52 ++++++++++++++++++++++++++ drivers/pcie/starfive/pcie.c | 26 +++++-------- examples/pcie/board/star64/pcie.system | 2 +- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/drivers/pcie/starfive/nvme.h b/drivers/pcie/starfive/nvme.h index f7f6ad399..26900d576 100644 --- a/drivers/pcie/starfive/nvme.h +++ b/drivers/pcie/starfive/nvme.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + /* References: @@ -9,3 +12,52 @@ [NVMEe-Transport-PCIe-1.1] NVMe over PCIe Transport Specification, Revision 1.1 (Aug 5, 2024) https://nvmexpress.org/wp-content/uploads/NVM-Express-PCI-Express-Transport-Specification-Revision-1.1-2024.08.05-Ratified.pdf */ + +/* [NVMe-2.1] Section 3.1.4 Controller Properties + RO = Read Only + RW = Read Write + RWC = Read/Write '1' to clear + RWS = Read/Write '1' to set +*/ +typedef struct nvme_controller { + uint64_t cap; /* Controller Capabilities (RO) */ + struct vs { + uint16_t mjr; /* Major Version */ + uint8_t mnr; /* Minor Version */ + uint8_t ter; /* Tertiary Version */ + } vs; /* Version (RO) */ + uint32_t intms; /* Interrupt Mask Set (RWS) */ + uint32_t intmc; /* Interrupt Mask Clear (RWC) */ + uint32_t cc; /* Controller Configuration (RW) */ + uint32_t _reserved; + uint32_t csts; /* Controller Status */ + uint32_t nssr; /* NVM Subsystem Reset (optional) */ + uint32_t aqa; /* Admin Queue Attributes */ + uint64_t asq; /* Admin Submission Queue Base Address */ + uint64_t acq; /* Admin Completion Queue Base Address */ + + uint32_t cmbloc; /* Controller Memory Buffer Location (optional) */ + uint32_t cmbsz; /* Controller Memory Buffer Size (optional) */ + uint32_t bpinfo; /* Boot Partition Information (optional) */ + uint32_t bprsel; /* Boot Partition Read Select (optional) */ + uint64_t bpmbl; /* Boot Partition Memory Buffer Location (optional) */ + uint64_t cmbmsc; /* Controller Memory Buffer Memory Space Control (optional) */ + uint32_t cmbsts; /* Controller Memory Buffer Status (optional) */ + uint32_t cmbebs; /* Controller Memory Buffer Elasticity Buffer Size (optional) */ + uint32_t cmbswtp; /* Controller Memory Buffer Sustained Write Throughput (optional) */ + uint32_t nssd; /* NVM Subsystem Shutdown (optional) */ + + uint32_t crto; /* Controller Ready Timeouts */ + uint32_t _reserved2; +} nvme_controller_t; + +_Static_assert(offsetof(nvme_controller_t, _reserved2) == 0x6C, "nvme_controller_t must match spec layout"); + +/* [NVMe-2.1] 3.1.4.5 Offset 14h: CC – Controller Configuration */ +#define NVME_CC_EN BIT(0) /* Controller Enable (RW) */ + +/* [NVMe-2.1] 3.1.4.6 Offset 1Ch: CSTS – Controller Status */ +#define NVME_CSTS_RDY BIT(0) /* Controller Ready (RO) */ + +/* [NVMe-2.1] 3.1.4.7 Offset 20h: NSSR – NVM Subsystem Reset */ +#define NVME_NSSRC_VALUE (0x4E564D65) /* NVM Subsystem Reset Control - Reset value */ diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index b73474701..9f6917d59 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -9,6 +9,7 @@ #include #include "pcie.h" +#include "nvme.h" // void *pcie_regs; @@ -18,7 +19,7 @@ uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x1000000 -uintptr_t nvme_regs; +volatile nvme_controller_t *nvme_controller; /* bus between [0, 256) device between [0, 31) @@ -83,14 +84,14 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) if (bar & BIT(0)) { sddf_dprintf("\tbase address for I/O\n"); - sddf_dprintf("\taddress: 0x%08x\n", bar & ~(BIT(0) | BIT(1))); + sddf_dprintf("\taddress: 0x%08x\n", (uint32_t)(bar & ~(BIT(0) | BIT(1)))); } else { sddf_dprintf("\tbase address for memory\n"); sddf_dprintf("\ttype: "); switch ((bar & (BIT(1) | BIT(2))) >> 1) { case 0b00: sddf_dprintf("32-bit space\n"); - sddf_dprintf("\tfull address: 0x%08x\n", bar & ~(BIT(4) - 1)); + sddf_dprintf("\tfull address: 0x%08x\n", (uint32_t)(bar & ~(BIT(4) - 1))); break; case 0b10: @@ -102,7 +103,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) uint32_t bar_upper = type0_header->base_address_registers[i + 1]; - sddf_dprintf("\tfull address: 0x%08x_%08x\n", bar_upper, bar & ~(BIT(4) - 1)); + sddf_dprintf("\tfull address: 0x%08x_%08x\n", bar_upper, (uint32_t)(bar & ~(BIT(4) - 1))); /* [PCI-3.0] 6.2.5.1 Address Maps (Implementation Note) p227*/ @@ -148,7 +149,7 @@ void init() { sddf_dprintf("pcie driver starting!\n"); - for (uint8_t bus = 0; bus < 256; bus++) { + for (uint8_t bus = 0; bus <= 255; bus++) { for (uint8_t device = 0; device < 32; device++) { for (uint8_t function = 0; function < 8; function++) { uintptr_t offset = get_bdf_offset(bus, device, function); @@ -165,14 +166,6 @@ void init() sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); sddf_dprintf("Starting NVME config...\n"); - // [NVME-2.1] Figure 33 (§3.1.4) - volatile struct { - uint64_t cap; - uint32_t vs; - uint32_t intms; - uint32_t intmc; - uint32_t cc; - } *nvme_reg = (void*)nvme_regs; // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 @@ -180,10 +173,9 @@ void init() // [NVME-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) - sddf_dprintf("CAP: %016lx\n", nvme_reg->cap); - sddf_dprintf("VS: %08x\n", nvme_reg->vs); - sddf_dprintf("CC: %08x\n", nvme_reg->cc); - + sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); + sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, nvme_controller->vs.ter); + sddf_dprintf("CC: %08x\n", nvme_controller->cc); } void notified(microkit_channel ch) diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 05bdbe5c2..9cd989b41 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -16,7 +16,7 @@ - + From d7ee9078c7f8b70d26f396920f9391000c810733 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 23 Sep 2024 11:55:15 +1000 Subject: [PATCH 12/60] dodgy ready admin controller? Signed-off-by: julia --- drivers/pcie/starfive/nvme.h | 24 +++++++++++++++--- drivers/pcie/starfive/pcie.c | 49 ++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/drivers/pcie/starfive/nvme.h b/drivers/pcie/starfive/nvme.h index 26900d576..dedc7788c 100644 --- a/drivers/pcie/starfive/nvme.h +++ b/drivers/pcie/starfive/nvme.h @@ -22,9 +22,9 @@ typedef struct nvme_controller { uint64_t cap; /* Controller Capabilities (RO) */ struct vs { - uint16_t mjr; /* Major Version */ - uint8_t mnr; /* Minor Version */ uint8_t ter; /* Tertiary Version */ + uint8_t mnr; /* Minor Version */ + uint16_t mjr; /* Major Version */ } vs; /* Version (RO) */ uint32_t intms; /* Interrupt Mask Set (RWS) */ uint32_t intmc; /* Interrupt Mask Clear (RWC) */ @@ -53,11 +53,29 @@ typedef struct nvme_controller { _Static_assert(offsetof(nvme_controller_t, _reserved2) == 0x6C, "nvme_controller_t must match spec layout"); +#define _LEN(start, end) ((end - start) + 1) +#define _MASK(start, end) ((BIT(_LEN(start, end)) - 1) << (start)) + +/* [NVMe-2.1] 3.1.4.1 Offset 0h: CAP – Controller Capabilities */ +#define NVME_CAP_NOIOCSS BIT(37 + 7) /* No I/O Command Set Support */ +#define NVME_CAP_IOCSS BIT(37 + 6) /* I/O Command Set Support */ +#define NVME_CAP_NCSS BIT(37 + 0) /* NVM Command Set Support */ + /* [NVMe-2.1] 3.1.4.5 Offset 14h: CC – Controller Configuration */ -#define NVME_CC_EN BIT(0) /* Controller Enable (RW) */ +#define NVME_CC_MPS_SHIFT 7 /* Host Memory Page Size */ +#define NVME_CC_MPS_MASK _MASK(7, 10) /* Host Memory Page Size */ +#define NVME_CC_CSS_SHIFT 4 /* I/O Command Set Selected */ +#define NVME_CC_CSS_MASK _MASK(4, 6) /* I/O Command Set Selected */ +#define NVME_CC_EN BIT(0) /* Controller Enable */ /* [NVMe-2.1] 3.1.4.6 Offset 1Ch: CSTS – Controller Status */ #define NVME_CSTS_RDY BIT(0) /* Controller Ready (RO) */ /* [NVMe-2.1] 3.1.4.7 Offset 20h: NSSR – NVM Subsystem Reset */ #define NVME_NSSRC_VALUE (0x4E564D65) /* NVM Subsystem Reset Control - Reset value */ + +/* [NVMe-2.1] 3.1.4.8 Offset 24h: AQA – Admin Queue Attributes */ +#define NVME_AQA_ACQS_SHIFT 16 /* Admin Completion Queue Size (#entries) */ +#define NVME_AQA_ACQS_MASK _MASK(16, 27) /* Admin Completion Queue Size (#entries) */ +#define NVME_AQA_ASQS_SHIFT 0 /* Admin Submission Queue Size (#entries) */ +#define NVME_AQA_ASQS_MASK _MASK(0, 11) /* Admin Submission Queue Size (#entries) */ diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 9f6917d59..a220ec67d 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -70,7 +70,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("header type: 0x%02x\n", header->header_type); sddf_dprintf("\thas multi-functions: %s\n", header->header_type & PCIE_HEADER_TYPE_HAS_MULTI_FUNCTIONS ? "yes" : "no"); - sddf_dprintf("\tlayout variant: 0x%02x\n", header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK); + sddf_dprintf("\tlayout variant: 0x%02lx\n", header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK); if ((header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK) == PCIE_HEADER_TYPE_GENERAL) { volatile pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; @@ -84,14 +84,14 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) if (bar & BIT(0)) { sddf_dprintf("\tbase address for I/O\n"); - sddf_dprintf("\taddress: 0x%08x\n", (uint32_t)(bar & ~(BIT(0) | BIT(1)))); + sddf_dprintf("\taddress: 0x%08lx\n", bar & ~(BIT(0) | BIT(1))); } else { sddf_dprintf("\tbase address for memory\n"); sddf_dprintf("\ttype: "); switch ((bar & (BIT(1) | BIT(2))) >> 1) { case 0b00: sddf_dprintf("32-bit space\n"); - sddf_dprintf("\tfull address: 0x%08x\n", (uint32_t)(bar & ~(BIT(4) - 1))); + sddf_dprintf("\tfull address: 0x%08lx\n", bar & ~(BIT(4) - 1)); break; case 0b10: @@ -103,7 +103,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) uint32_t bar_upper = type0_header->base_address_registers[i + 1]; - sddf_dprintf("\tfull address: 0x%08x_%08x\n", bar_upper, (uint32_t)(bar & ~(BIT(4) - 1))); + sddf_dprintf("\tfull address: 0x%08x_%08lx\n", bar_upper, bar & ~(BIT(4) - 1)); /* [PCI-3.0] 6.2.5.1 Address Maps (Implementation Note) p227*/ @@ -171,11 +171,50 @@ void init() // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 - // [NVME-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, nvme_controller->vs.ter); sddf_dprintf("CC: %08x\n", nvme_controller->cc); + + /* [NVME-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ + nvme_controller->cc &= ~NVME_CC_EN; + + // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) + while (nvme_controller->csts & NVME_CSTS_RDY); + + // 2. Configure Admin Queue(s) TODO. + // nvme_controller->asq = nvme_admin_submission_queue; + // nvme_controller->acq = nvme_admin_completion_queue; + /* TODO: queue size def. */ + nvme_controller->aqa &= ~(NVME_AQA_ACQS_MASK | NVME_AQA_ASQS_MASK); + nvme_controller->aqa |= ((4096 - 1) << NVME_AQA_ACQS_SHIFT) | ((4096 - 1) << NVME_AQA_ASQS_SHIFT); + + // 3. Initialise Command Support Sets. + nvme_controller->cc &= ~(NVME_CC_CSS_MASK); + if (nvme_controller->cap & NVME_CAP_NOIOCSS) { + nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_IOCSS) { + nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_NCSS) { + nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; + } + + // (not-spec but in vroom?) TODO set completion/submission entry sizes? + + // 4a. Arbitration Mechanism (TODO) + // 4b. Memory Page Size + nvme_controller->cc &= ~NVME_CC_MPS_MASK; + /* nb: host page size. TODO: do we have a define for this? */ + /* n.b.: page size = 2 ^ (12 + MPS)*/ + nvme_controller->cc |= (4096 >> 12) << NVME_CC_MPS_SHIFT; + + // 5. Enable the controller + nvme_controller->cc |= NVME_CC_EN; + + // 6. Wait for ready + sddf_dprintf("waiting ready\n"); + while (!(nvme_controller->csts & NVME_CSTS_RDY)); + sddf_dprintf("now ready\n"); } void notified(microkit_channel ch) From b0b8cc35f246d45170ba0ba88a9aecf6e81cb20e Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 27 Sep 2024 13:55:39 +1000 Subject: [PATCH 13/60] Basic working NVMe queue ready Signed-off-by: julia --- drivers/pcie/starfive/nvme.h | 129 +++++++++++++++- drivers/pcie/starfive/pcie.c | 195 ++++++++++++++++--------- examples/pcie/board/star64/pcie.system | 5 + 3 files changed, 254 insertions(+), 75 deletions(-) diff --git a/drivers/pcie/starfive/nvme.h b/drivers/pcie/starfive/nvme.h index dedc7788c..9b58d6573 100644 --- a/drivers/pcie/starfive/nvme.h +++ b/drivers/pcie/starfive/nvme.h @@ -57,9 +57,11 @@ _Static_assert(offsetof(nvme_controller_t, _reserved2) == 0x6C, "nvme_controller #define _MASK(start, end) ((BIT(_LEN(start, end)) - 1) << (start)) /* [NVMe-2.1] 3.1.4.1 Offset 0h: CAP – Controller Capabilities */ -#define NVME_CAP_NOIOCSS BIT(37 + 7) /* No I/O Command Set Support */ -#define NVME_CAP_IOCSS BIT(37 + 6) /* I/O Command Set Support */ -#define NVME_CAP_NCSS BIT(37 + 0) /* NVM Command Set Support */ +#define NVME_CAP_NOIOCSS BIT(37 + 7) /* No I/O Command Set Support */ +#define NVME_CAP_IOCSS BIT(37 + 6) /* I/O Command Set Support */ +#define NVME_CAP_NCSS BIT(37 + 0) /* NVM Command Set Support */ +#define NVME_CAP_DSTRD_SHIFT 32 /* Doorbell Stride (2 ^ (2 + DSTRD)) */ +#define NVME_CAP_DSTRD_MASK _MASK(32, 35) /* Doorbell Stride (2 ^ (2 + DSTRD)) */ /* [NVMe-2.1] 3.1.4.5 Offset 14h: CC – Controller Configuration */ #define NVME_CC_MPS_SHIFT 7 /* Host Memory Page Size */ @@ -79,3 +81,124 @@ _Static_assert(offsetof(nvme_controller_t, _reserved2) == 0x6C, "nvme_controller #define NVME_AQA_ACQS_MASK _MASK(16, 27) /* Admin Completion Queue Size (#entries) */ #define NVME_AQA_ASQS_SHIFT 0 /* Admin Submission Queue Size (#entries) */ #define NVME_AQA_ASQS_MASK _MASK(0, 11) /* Admin Submission Queue Size (#entries) */ + +/** + * Queue Structures + */ + +/* [NVMe-2.1] 4.1 Submission Queue Entry */ +typedef struct nvme_submission_queue_entry { + // TODO: split out to opcode etc (figure 91??) + + uint32_t cdw0; /* Command Dword 0 (common) */ + uint32_t nsid; /* Namespace Identifier */ + uint32_t cdw2; /* Command Dword 2 (command-specific) */ + uint32_t cdw3; /* Command Dword 3 (command-specific) */ + uint64_t mptr; /* Metadata Pointer */ + uint64_t dptr_lo; /* Data Pointer (low 8 bytes) */ + uint64_t dptr_hi; /* Data Pointer (high 8 bytes) */ + uint32_t cdw10; /* Command Dword 10 (command-specific) */ + uint32_t cdw11; /* Command Dword 11 (command-specific) */ + uint32_t cdw12; /* Command Dword 12 (command-specific) */ + uint32_t cdw13; /* Command Dword 13 (command-specific) */ + uint32_t cdw14; /* Command Dword 14 (command-specific) */ + uint32_t cdw15; /* Command Dword 15 (command-specific) */ +} nvme_submission_queue_entry_t; +_Static_assert(sizeof(nvme_submission_queue_entry_t) == 64, "Each Common Command Format command is 64 bytes in size."); + +/* [NVMe-2.1] 4.2 Completion Queue Entry */ +typedef struct nvme_completion_queue_entry { + uint32_t cdw0; /* Command Dword 0 (command-specific) */ + uint32_t cdw1; /* Command Dword 1 (command-specific) */ + uint16_t sqhd; /* Submission Queue Head Pointer */ + uint16_t sqid; /* Submission Queue ID */ + uint16_t cid; /* Command Identifier */ + uint16_t phase_tag_and_status; /* Phase Tag (P) and Status */ +} nvme_completion_queue_entry_t; +_Static_assert(sizeof(nvme_completion_queue_entry_t) == 16, + "The Common Completion Queue Entry Layout is 16 bytes in size"); + +/** + * Below here is NVMe PCIe Transport Specific Properties. + */ + +#define NVME_PCIE_SQT_MASK _MASK(0, 15) /* Submission Queue Tail*/ +#define NVME_PCIE_CQH_MASK _MASK(0, 15) /* Completion Queue Head */ + +#define NVME_PCIE_DOORBELL_OFFSET(i, DSTRD) (0x1000 + (i * (4 << DSTRD))) + +/** + * Get the value of the 'Submission Queue y Tail Doorbell' register. + * + * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL + * + * @param nvme_controller Base address of the NVMe controller registers + * @param DSTRD CAP.DSTRD of the NVMe controller + * @param y the submission queue number. the 0th queue is the admin queue. + * + * @todo how big can y be? + */ +static inline uint16_t nvme_sqytdbl_read(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y) +{ + volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y, DSTRD); + return (*addr) & NVME_PCIE_SQT_MASK; +} + +/** + * Set the value of the 'Submission Queue y Tail Doorbell' register. + * + * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL + * + * @param nvme_controller Base address of the NVMe controller registers + * @param DSTRD CAP.DSTRD of the NVMe controller + * @param y the submission queue number. the 0th queue is the admin queue. + * @param tail the new value of the tail register + * + * @todo how big can y be? + */ +static inline void nvme_sqytdbl_write(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y, uint16_t tail) +{ + volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y, DSTRD); + uint32_t val = *addr; + val &= ~NVME_PCIE_SQT_MASK; + val |= tail & NVME_PCIE_SQT_MASK; + *addr = val; +} + +/** + * Get the value of the 'Completion Queue y Head Doorbell' register. + * + * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.2 CQyHDBL + * + * @param nvme_controller Base address of the NVMe controller registers + * @param DSTRD CAP.DSTRD of the NVMe controller + * @param y the completion queue number. the 0th queue is the admin queue. + * + * @todo how big can y be? + */ +static inline uint16_t nvme_cqyhdbl_read(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y) +{ + volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y + 1, DSTRD); + return (*addr) & NVME_PCIE_CQH_MASK; +} + +/** + * Set the value of the 'Completion Queue y Head Doorbell' register. + * + * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.2 CQyHDBL + * + * @param nvme_controller Base address of the NVMe controller registers + * @param DSTRD CAP.DSTRD of the NVMe controller + * @param y the submission queue number. the 0th queue is the admin queue. + * @param head the new value of the head register + * + * @todo how big can y be? + */ +static inline void nvme_cqyhdbl_write(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y, uint16_t head) +{ + volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y + 1, DSTRD); + uint32_t val = *addr; + val &= ~NVME_PCIE_CQH_MASK; + val |= head & NVME_PCIE_CQH_MASK; + *addr = val; +} diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index a220ec67d..9157eafca 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -20,6 +20,17 @@ uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x1000000 volatile nvme_controller_t *nvme_controller; +nvme_submission_queue_entry_t *nvme_asq_region; +nvme_completion_queue_entry_t *nvme_acq_region; +uintptr_t nvme_asq_region_paddr; +uintptr_t nvme_acq_region_paddr; +#define NVME_ADMIN_QUEUE_SIZE 0x1000 + +/* TODO: don't hardcode */ +#define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) +#define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) +_Static_assert(NVME_ASQ_CAPACITY <= 0x1000, "capacity of ASQ must be <=4096 (entries)"); +_Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (entries)"); /* bus between [0, 256) device between [0, 31) @@ -69,7 +80,8 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("base-class code: 0x%02x | sub-class code: 0x%02x\n", header->base_class_code, header->subclass_code); sddf_dprintf("header type: 0x%02x\n", header->header_type); - sddf_dprintf("\thas multi-functions: %s\n", header->header_type & PCIE_HEADER_TYPE_HAS_MULTI_FUNCTIONS ? "yes" : "no"); + sddf_dprintf("\thas multi-functions: %s\n", + header->header_type & PCIE_HEADER_TYPE_HAS_MULTI_FUNCTIONS ? "yes" : "no"); sddf_dprintf("\tlayout variant: 0x%02lx\n", header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK); if ((header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK) == PCIE_HEADER_TYPE_GENERAL) { @@ -89,55 +101,54 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("\tbase address for memory\n"); sddf_dprintf("\ttype: "); switch ((bar & (BIT(1) | BIT(2))) >> 1) { - case 0b00: - sddf_dprintf("32-bit space\n"); - sddf_dprintf("\tfull address: 0x%08lx\n", bar & ~(BIT(4) - 1)); - break; + case 0b00: + sddf_dprintf("32-bit space\n"); + sddf_dprintf("\tfull address: 0x%08lx\n", bar & ~(BIT(4) - 1)); + break; - case 0b10: - sddf_dprintf("64-bit space\n"); - if (i >= 5) { - sddf_dprintf("\tspecified 64-bit in the last slot, ignoring..."); - continue; - } + case 0b10: + sddf_dprintf("64-bit space\n"); + if (i >= 5) { + sddf_dprintf("\tspecified 64-bit in the last slot, ignoring..."); + continue; + } - uint32_t bar_upper = type0_header->base_address_registers[i + 1]; + uint32_t bar_upper = type0_header->base_address_registers[i + 1]; - sddf_dprintf("\tfull address: 0x%08x_%08lx\n", bar_upper, bar & ~(BIT(4) - 1)); + sddf_dprintf("\tfull address: 0x%08x_%08lx\n", bar_upper, bar & ~(BIT(4) - 1)); /* [PCI-3.0] 6.2.5.1 Address Maps (Implementation Note) p227*/ - // Decode (I/O or memory) of a register is disabled via the command register before sizing a Base Address register. - header->command &= ~(BIT(1)); + // Decode (I/O or memory) of a register is disabled via the command register before sizing a Base Address register. + header->command &= ~(BIT(1)); - // calculate size. - type0_header->base_address_registers[i] = 0xffffffff; - type0_header->base_address_registers[i + 1] = 0xffffffff; + // calculate size. + type0_header->base_address_registers[i] = 0xffffffff; + type0_header->base_address_registers[i + 1] = 0xffffffff; - // read back - uint32_t size_lower = type0_header->base_address_registers[i]; - uint32_t size_upper = type0_header->base_address_registers[i + 1]; - uint64_t size_readback = ((uint64_t)size_upper << 32) | (size_lower); - // calculation can be done from the 32-bit value read by first clearing encoding information bits - // (bit 0 for I/O, bits 0-3 formemory), inverting all 32 bits (logical NOT), then incrementing by 1 - size_readback &= ~(BIT(3) | BIT(2) | BIT(1) | BIT(0)); - size_readback = ~size_readback; - size_readback += 1; + // read back + uint32_t size_lower = type0_header->base_address_registers[i]; + uint32_t size_upper = type0_header->base_address_registers[i + 1]; + uint64_t size_readback = ((uint64_t)size_upper << 32) | (size_lower); + // calculation can be done from the 32-bit value read by first clearing encoding information bits + // (bit 0 for I/O, bits 0-3 formemory), inverting all 32 bits (logical NOT), then incrementing by 1 + size_readback &= ~(BIT(3) | BIT(2) | BIT(1) | BIT(0)); + size_readback = ~size_readback; + size_readback += 1; - sddf_dprintf("\tsize: 0x%lx\n", size_readback); + sddf_dprintf("\tsize: 0x%lx\n", size_readback); - // The original value in the Base Address register is restored before re-enabling - // decode in the command register of the device. - type0_header->base_address_registers[i] = bar; - type0_header->base_address_registers[i + 1] = bar_upper; - header->command |= BIT(1); + // The original value in the Base Address register is restored before re-enabling + // decode in the command register of the device. + type0_header->base_address_registers[i] = bar; + type0_header->base_address_registers[i + 1] = bar_upper; + header->command |= BIT(1); + i += 1; // skip one slot. + break; - i += 1; // skip one slot. - break; - - default: - sddf_dprintf("reserved\n"); + default: + sddf_dprintf("reserved\n"); } sddf_dprintf("\tprefetchable: %s\n", bar & BIT(3) ? "yes" : "no"); } @@ -145,49 +156,32 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) } } -void init() +/* [NVMe-2.1] 3.3.1 Memory-based Transport Queue Model (PCIe) */ +void nvme_queue_init() { - sddf_dprintf("pcie driver starting!\n"); - - for (uint8_t bus = 0; bus <= 255; bus++) { - for (uint8_t device = 0; device < 32; device++) { - for (uint8_t function = 0; function < 8; function++) { - uintptr_t offset = get_bdf_offset(bus, device, function); - if (offset >= PCIE_CONFIG_SIZE) { - goto out; - } - - device_print(bus, device, function); - } - } - } - -out: - sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); - - sddf_dprintf("Starting NVME config...\n"); - - // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 - - // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 - + // TODO. +} +/* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ +void nvme_controller_init() +{ sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); - sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, nvme_controller->vs.ter); + sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + nvme_controller->vs.ter); sddf_dprintf("CC: %08x\n", nvme_controller->cc); - /* [NVME-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ + nvme_controller->cc &= ~NVME_CC_EN; // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) while (nvme_controller->csts & NVME_CSTS_RDY); // 2. Configure Admin Queue(s) TODO. - // nvme_controller->asq = nvme_admin_submission_queue; - // nvme_controller->acq = nvme_admin_completion_queue; - /* TODO: queue size def. */ + nvme_controller->asq = nvme_asq_region_paddr; + nvme_controller->acq = nvme_acq_region_paddr; nvme_controller->aqa &= ~(NVME_AQA_ACQS_MASK | NVME_AQA_ASQS_MASK); - nvme_controller->aqa |= ((4096 - 1) << NVME_AQA_ACQS_SHIFT) | ((4096 - 1) << NVME_AQA_ASQS_SHIFT); + nvme_controller->aqa |= ((NVME_ASQ_CAPACITY - 1) << NVME_AQA_ASQS_SHIFT) + | ((NVME_ACQ_CAPACITY - 1) << NVME_AQA_ACQS_SHIFT); // 3. Initialise Command Support Sets. nvme_controller->cc &= ~(NVME_CC_CSS_MASK); @@ -199,8 +193,6 @@ void init() nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; } - // (not-spec but in vroom?) TODO set completion/submission entry sizes? - // 4a. Arbitration Mechanism (TODO) // 4b. Memory Page Size nvme_controller->cc &= ~NVME_CC_MPS_MASK; @@ -214,9 +206,68 @@ void init() // 6. Wait for ready sddf_dprintf("waiting ready\n"); while (!(nvme_controller->csts & NVME_CSTS_RDY)); - sddf_dprintf("now ready\n"); + + // 7. Send the Identify Controller command (CNS = 01h) + uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; + uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); + uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); + sddf_dprintf("admin tail: %u\n", sq_tail); + sddf_dprintf("admin head: %u\n", cq_head); + + nvme_asq_region[sq_tail] = (nvme_submission_queue_entry_t){ + .cdw0 = 01, + }; + + // todo: overflow? + nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); + sddf_dprintf("new tail: %u\n", sq_tail); + + uint16_t orig_cq_head = cq_head; + while (cq_head == orig_cq_head) { + cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); + } + + sddf_dprintf("new head: %u\n", cq_head); +} + +void nvme_init() +{ + sddf_dprintf("Starting NVME config...\n"); + + // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 + + // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 + + nvme_controller_init(); + nvme_queue_init(); +} + +void init() +{ + sddf_dprintf("pcie driver starting!\n"); + + for (uint8_t bus = 0; bus <= 255; bus++) { + for (uint8_t device = 0; device < 32; device++) { + for (uint8_t function = 0; function < 8; function++) { + uintptr_t offset = get_bdf_offset(bus, device, function); + if (offset >= PCIE_CONFIG_SIZE) { + goto out; + } + + device_print(bus, device, function); + } + } + } + +out: + sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); + + nvme_init(); } +/* See: 3.3 NVM Queue MOdels */ +// CAP.MQES + void notified(microkit_channel ch) { } diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 9cd989b41..335cb63da 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -11,12 +11,17 @@ + + + + + From 1a9919fea24384210ddf98803571f9634ae2a754 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 27 Sep 2024 14:10:09 +1000 Subject: [PATCH 14/60] add a page size define Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 2 +- include/sddf/util/util.h | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 9157eafca..ce263c604 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -198,7 +198,7 @@ void nvme_controller_init() nvme_controller->cc &= ~NVME_CC_MPS_MASK; /* nb: host page size. TODO: do we have a define for this? */ /* n.b.: page size = 2 ^ (12 + MPS)*/ - nvme_controller->cc |= (4096 >> 12) << NVME_CC_MPS_SHIFT; + nvme_controller->cc |= (PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT; // 5. Enable the controller nvme_controller->cc |= NVME_CC_EN; diff --git a/include/sddf/util/util.h b/include/sddf/util/util.h index ea743883e..aa2f4cdbd 100644 --- a/include/sddf/util/util.h +++ b/include/sddf/util/util.h @@ -67,6 +67,11 @@ void _assert_fail(const char *assertion, const char *file, unsigned int line, #endif #endif +#ifndef PAGE_SIZE +#define PAGE_SIZE_4K 0x1000 +#define PAGE_SIZE PAGE_SIZE_4K +#endif + static inline int sddf_isspace(int ch) { return ch == ' ' || ch == '\f' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\v'; From a8853ba67a43b4ded6e491a7a9a219e3227f78c8 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 27 Sep 2024 14:48:14 +1000 Subject: [PATCH 15/60] try to get a actual command Signed-off-by: julia --- drivers/pcie/starfive/nvme.h | 3 +++ drivers/pcie/starfive/pcie.c | 21 +++++++++++++++------ examples/pcie/board/star64/pcie.system | 5 +++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/drivers/pcie/starfive/nvme.h b/drivers/pcie/starfive/nvme.h index 9b58d6573..c513cdc33 100644 --- a/drivers/pcie/starfive/nvme.h +++ b/drivers/pcie/starfive/nvme.h @@ -118,6 +118,9 @@ typedef struct nvme_completion_queue_entry { _Static_assert(sizeof(nvme_completion_queue_entry_t) == 16, "The Common Completion Queue Entry Layout is 16 bytes in size"); + +#define NVME_ADMIN_COMMAND_IDENTIFY 0x6 + /** * Below here is NVMe PCIe Transport Specific Properties. */ diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index ce263c604..466a12570 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -26,6 +26,10 @@ uintptr_t nvme_asq_region_paddr; uintptr_t nvme_acq_region_paddr; #define NVME_ADMIN_QUEUE_SIZE 0x1000 + +uintptr_t data_region_paddr; +uint8_t *data_region; + /* TODO: don't hardcode */ #define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) #define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) @@ -196,9 +200,8 @@ void nvme_controller_init() // 4a. Arbitration Mechanism (TODO) // 4b. Memory Page Size nvme_controller->cc &= ~NVME_CC_MPS_MASK; - /* nb: host page size. TODO: do we have a define for this? */ - /* n.b.: page size = 2 ^ (12 + MPS)*/ - nvme_controller->cc |= (PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT; + /* n.b. page size = 2 ^ (12 + MPS) */ + nvme_controller->cc |= ((PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT) & NVME_CC_MPS_MASK; // 5. Enable the controller nvme_controller->cc |= NVME_CC_EN; @@ -207,20 +210,23 @@ void nvme_controller_init() sddf_dprintf("waiting ready\n"); while (!(nvme_controller->csts & NVME_CSTS_RDY)); - // 7. Send the Identify Controller command (CNS = 01h) + // 7. Send the Identify Controller command (Identify with CNS = 01h) uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); sddf_dprintf("admin tail: %u\n", sq_tail); sddf_dprintf("admin head: %u\n", cq_head); + // Identify Command (5.1.13) nvme_asq_region[sq_tail] = (nvme_submission_queue_entry_t){ - .cdw0 = 01, + .cdw0 = /* CID */ (sq_tail << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, + .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, + .dptr_hi = 0, + .dptr_lo = data_region_paddr, /* TEMP */ }; // todo: overflow? nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); - sddf_dprintf("new tail: %u\n", sq_tail); uint16_t orig_cq_head = cq_head; while (cq_head == orig_cq_head) { @@ -228,6 +234,9 @@ void nvme_controller_init() } sddf_dprintf("new head: %u\n", cq_head); + for (int i = 0; i < 32; i++) { + sddf_dprintf("[%04x]: %x\n", i, data_region[i]); + } } void nvme_init() diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 335cb63da..4194c3ceb 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -14,6 +14,8 @@ + + @@ -23,6 +25,9 @@ + + + From 961c8e331170ccd36c6ab9721fd447e3ebdb02f9 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 30 Sep 2024 11:25:04 +1000 Subject: [PATCH 16/60] actually get admin queue commands working Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 56 +++++++++++++++++++++----- examples/pcie/board/star64/pcie.system | 6 ++- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 466a12570..d30387b3f 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -20,8 +20,8 @@ uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x1000000 volatile nvme_controller_t *nvme_controller; -nvme_submission_queue_entry_t *nvme_asq_region; -nvme_completion_queue_entry_t *nvme_acq_region; +volatile nvme_submission_queue_entry_t *nvme_asq_region; +volatile nvme_completion_queue_entry_t *nvme_acq_region; uintptr_t nvme_asq_region_paddr; uintptr_t nvme_acq_region_paddr; #define NVME_ADMIN_QUEUE_SIZE 0x1000 @@ -166,6 +166,20 @@ void nvme_queue_init() // TODO. } +void nvme_debug_dump_controller_regs() +{ + sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); + sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + nvme_controller->vs.ter); + sddf_dprintf("INTMS: %08x\n", nvme_controller->intms); + sddf_dprintf("INTMC: %08x\n", nvme_controller->intmc); + sddf_dprintf("CC: %08x\n", nvme_controller->cc); + sddf_dprintf("CSTS: %08x\n", nvme_controller->csts); + sddf_dprintf("AQA: %08x\n", nvme_controller->aqa); + sddf_dprintf("ASQ: %016lx\n", nvme_controller->asq); + sddf_dprintf("ACQ: %016lx\n", nvme_controller->acq); +} + /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() { @@ -180,7 +194,9 @@ void nvme_controller_init() // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) while (nvme_controller->csts & NVME_CSTS_RDY); - // 2. Configure Admin Queue(s) TODO. + // 2. Configure Admin Queue(s) + assert(nvme_asq_region_paddr != 0x0); + assert(nvme_acq_region_paddr != 0x0); nvme_controller->asq = nvme_asq_region_paddr; nvme_controller->acq = nvme_acq_region_paddr; nvme_controller->aqa &= ~(NVME_AQA_ACQS_MASK | NVME_AQA_ASQS_MASK); @@ -203,12 +219,15 @@ void nvme_controller_init() /* n.b. page size = 2 ^ (12 + MPS) */ nvme_controller->cc |= ((PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT) & NVME_CC_MPS_MASK; + // TODO: See initialisation note under §4.2.4; fine since already that way. + // 5. Enable the controller nvme_controller->cc |= NVME_CC_EN; // 6. Wait for ready - sddf_dprintf("waiting ready\n"); + sddf_dprintf("waiting ready...\n"); while (!(nvme_controller->csts & NVME_CSTS_RDY)); + sddf_dprintf("\tdone\n"); // 7. Send the Identify Controller command (Identify with CNS = 01h) uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; @@ -219,23 +238,38 @@ void nvme_controller_init() // Identify Command (5.1.13) nvme_asq_region[sq_tail] = (nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (sq_tail << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, + .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, .dptr_hi = 0, .dptr_lo = data_region_paddr, /* TEMP */ }; + uint32_t orig_dw3 = ((volatile uint32_t *)nvme_acq_region)[3]; + sddf_dprintf("orig dw3: %x\n", orig_dw3); + // todo: overflow? nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); - uint16_t orig_cq_head = cq_head; - while (cq_head == orig_cq_head) { - cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); - } + nvme_debug_dump_controller_regs(); + + // 4.2.4.1 shows how to send commands + sddf_dprintf("waiting for completed command...\n") + while (!(nvme_acq_region[cq_head].phase_tag_and_status & BIT(0))); + sddf_dprintf("complete\n"); + + + // sddf_dprintf("new submission tail: %u\n", nvme_sqytdbl_read(nvme_controller, DSTRD, 0)); + // sddf_dprintf("waiting completion queue change...\n"); + // uint16_t orig_cq_head = cq_head; + // while (cq_head == orig_cq_head) { + // cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); + // } + // sddf_dprintf("\tdone\n"); sddf_dprintf("new head: %u\n", cq_head); - for (int i = 0; i < 32; i++) { - sddf_dprintf("[%04x]: %x\n", i, data_region[i]); + // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s + for (int i = 0; i < 64; i++) { + sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); } } diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 4194c3ceb..5aa41981f 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -23,10 +23,12 @@ + + - - + + From d002123e4e016acd947ba068258169128300dd64 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 30 Sep 2024 11:40:35 +1000 Subject: [PATCH 17/60] move nvme stuff (mostly) out of the pcie fifle Signed-off-by: julia --- build.zig | 15 +-- drivers/nvme/nvme.c | 145 ++++++++++++++++++++++++ drivers/{pcie/starfive => nvme}/nvme.h | 2 + drivers/pcie/starfive/pcie.c | 146 +------------------------ 4 files changed, 158 insertions(+), 150 deletions(-) create mode 100644 drivers/nvme/nvme.c rename drivers/{pcie/starfive => nvme}/nvme.h (99%) diff --git a/build.zig b/build.zig index b4b3784a8..560eed095 100644 --- a/build.zig +++ b/build.zig @@ -261,18 +261,19 @@ fn addPcieDriver( optimize: std.builtin.OptimizeMode, ) *std.Build.Step.Compile { const driver = addPd(b, .{ - .name = b.fmt("driver_pcie_{s}.elf", .{ @tagName(class) }), + .name = b.fmt("driver_pcie_{s}.elf", .{@tagName(class)}), .target = target, .optimize = optimize, .strip = false, }); - const source = b.fmt("drivers/pcie/{s}/pcie.c", .{ @tagName(class) }); - driver.addCSourceFile(.{ - .file = b.path(source), - .flags = &.{ "-Werror" } - }); - driver.addIncludePath(b.path(b.fmt("drivers/pcie/{s}/", .{ @tagName(class) }))); + const source = b.fmt("drivers/pcie/{s}/pcie.c", .{@tagName(class)}); + driver.addCSourceFile(.{ .file = b.path(source), .flags = &.{"-Werror"} }); + driver.addIncludePath(b.path(b.fmt("drivers/pcie/{s}/", .{@tagName(class)}))); driver.addIncludePath(b.path("include")); + + driver.addCSourceFile(.{ .file = b.path("drivers/nvme/nvme.c"), .flags = &.{"-Werror"} }); + driver.addIncludePath(b.path("drivers/nvme/")); + driver.linkLibrary(util); return driver; diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c new file mode 100644 index 000000000..1871c5f2b --- /dev/null +++ b/drivers/nvme/nvme.c @@ -0,0 +1,145 @@ +#include "nvme.h" + +#include + +volatile nvme_controller_t *nvme_controller; +volatile nvme_submission_queue_entry_t *nvme_asq_region; +volatile nvme_completion_queue_entry_t *nvme_acq_region; +uintptr_t nvme_asq_region_paddr; +uintptr_t nvme_acq_region_paddr; +#define NVME_ADMIN_QUEUE_SIZE 0x1000 + + +uintptr_t data_region_paddr; +uint8_t *data_region; + +/* TODO: don't hardcode */ +#define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) +#define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) +_Static_assert(NVME_ASQ_CAPACITY <= 0x1000, "capacity of ASQ must be <=4096 (entries)"); +_Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (entries)"); + +/* [NVMe-2.1] 3.3.1 Memory-based Transport Queue Model (PCIe) */ +void nvme_queue_init() +{ + // TODO. +} + +void nvme_debug_dump_controller_regs() +{ + sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); + sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + nvme_controller->vs.ter); + sddf_dprintf("INTMS: %08x\n", nvme_controller->intms); + sddf_dprintf("INTMC: %08x\n", nvme_controller->intmc); + sddf_dprintf("CC: %08x\n", nvme_controller->cc); + sddf_dprintf("CSTS: %08x\n", nvme_controller->csts); + sddf_dprintf("AQA: %08x\n", nvme_controller->aqa); + sddf_dprintf("ASQ: %016lx\n", nvme_controller->asq); + sddf_dprintf("ACQ: %016lx\n", nvme_controller->acq); +} + +/* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ +void nvme_controller_init() +{ + sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); + sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + nvme_controller->vs.ter); + sddf_dprintf("CC: %08x\n", nvme_controller->cc); + + + nvme_controller->cc &= ~NVME_CC_EN; + + // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) + while (nvme_controller->csts & NVME_CSTS_RDY); + + // 2. Configure Admin Queue(s) + assert(nvme_asq_region_paddr != 0x0); + assert(nvme_acq_region_paddr != 0x0); + nvme_controller->asq = nvme_asq_region_paddr; + nvme_controller->acq = nvme_acq_region_paddr; + nvme_controller->aqa &= ~(NVME_AQA_ACQS_MASK | NVME_AQA_ASQS_MASK); + nvme_controller->aqa |= ((NVME_ASQ_CAPACITY - 1) << NVME_AQA_ASQS_SHIFT) + | ((NVME_ACQ_CAPACITY - 1) << NVME_AQA_ACQS_SHIFT); + + // 3. Initialise Command Support Sets. + nvme_controller->cc &= ~(NVME_CC_CSS_MASK); + if (nvme_controller->cap & NVME_CAP_NOIOCSS) { + nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_IOCSS) { + nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_NCSS) { + nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; + } + + // 4a. Arbitration Mechanism (TODO) + // 4b. Memory Page Size + nvme_controller->cc &= ~NVME_CC_MPS_MASK; + /* n.b. page size = 2 ^ (12 + MPS) */ + nvme_controller->cc |= ((PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT) & NVME_CC_MPS_MASK; + + // TODO: See initialisation note under §4.2.4; fine since already that way. + + // 5. Enable the controller + nvme_controller->cc |= NVME_CC_EN; + + // 6. Wait for ready + sddf_dprintf("waiting ready...\n"); + while (!(nvme_controller->csts & NVME_CSTS_RDY)); + sddf_dprintf("\tdone\n"); + + // 7. Send the Identify Controller command (Identify with CNS = 01h) + uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; + uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); + uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); + sddf_dprintf("admin tail: %u\n", sq_tail); + sddf_dprintf("admin head: %u\n", cq_head); + + // Identify Command (5.1.13) + nvme_asq_region[sq_tail] = (nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, + .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, + .dptr_hi = 0, + .dptr_lo = data_region_paddr, /* TEMP */ + }; + + uint32_t orig_dw3 = ((volatile uint32_t *)nvme_acq_region)[3]; + sddf_dprintf("orig dw3: %x\n", orig_dw3); + + // todo: overflow? + nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); + + nvme_debug_dump_controller_regs(); + + // 4.2.4.1 shows how to send commands + sddf_dprintf("waiting for completed command...\n"); + while (!(nvme_acq_region[cq_head].phase_tag_and_status & BIT(0))); + sddf_dprintf("complete\n"); + + + // sddf_dprintf("new submission tail: %u\n", nvme_sqytdbl_read(nvme_controller, DSTRD, 0)); + // sddf_dprintf("waiting completion queue change...\n"); + // uint16_t orig_cq_head = cq_head; + // while (cq_head == orig_cq_head) { + // cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); + // } + // sddf_dprintf("\tdone\n"); + + sddf_dprintf("new head: %u\n", cq_head); + // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s + for (int i = 0; i < 64; i++) { + sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); + } +} + +void nvme_init() +{ + sddf_dprintf("Starting NVME config...\n"); + + // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 + + // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 + + nvme_controller_init(); + nvme_queue_init(); +} diff --git a/drivers/pcie/starfive/nvme.h b/drivers/nvme/nvme.h similarity index 99% rename from drivers/pcie/starfive/nvme.h rename to drivers/nvme/nvme.h index c513cdc33..3f679c3f8 100644 --- a/drivers/pcie/starfive/nvme.h +++ b/drivers/nvme/nvme.h @@ -3,6 +3,8 @@ #include #include +#include + /* References: diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index d30387b3f..9b93ff136 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -9,7 +9,9 @@ #include #include "pcie.h" -#include "nvme.h" + +// TODO: Separate components or something. +void nvme_init(); // void *pcie_regs; @@ -19,23 +21,6 @@ uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x1000000 -volatile nvme_controller_t *nvme_controller; -volatile nvme_submission_queue_entry_t *nvme_asq_region; -volatile nvme_completion_queue_entry_t *nvme_acq_region; -uintptr_t nvme_asq_region_paddr; -uintptr_t nvme_acq_region_paddr; -#define NVME_ADMIN_QUEUE_SIZE 0x1000 - - -uintptr_t data_region_paddr; -uint8_t *data_region; - -/* TODO: don't hardcode */ -#define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) -#define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) -_Static_assert(NVME_ASQ_CAPACITY <= 0x1000, "capacity of ASQ must be <=4096 (entries)"); -_Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (entries)"); - /* bus between [0, 256) device between [0, 31) function between [0, 8) @@ -160,131 +145,6 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) } } -/* [NVMe-2.1] 3.3.1 Memory-based Transport Queue Model (PCIe) */ -void nvme_queue_init() -{ - // TODO. -} - -void nvme_debug_dump_controller_regs() -{ - sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); - sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, - nvme_controller->vs.ter); - sddf_dprintf("INTMS: %08x\n", nvme_controller->intms); - sddf_dprintf("INTMC: %08x\n", nvme_controller->intmc); - sddf_dprintf("CC: %08x\n", nvme_controller->cc); - sddf_dprintf("CSTS: %08x\n", nvme_controller->csts); - sddf_dprintf("AQA: %08x\n", nvme_controller->aqa); - sddf_dprintf("ASQ: %016lx\n", nvme_controller->asq); - sddf_dprintf("ACQ: %016lx\n", nvme_controller->acq); -} - -/* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ -void nvme_controller_init() -{ - sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); - sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, - nvme_controller->vs.ter); - sddf_dprintf("CC: %08x\n", nvme_controller->cc); - - - nvme_controller->cc &= ~NVME_CC_EN; - - // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) - while (nvme_controller->csts & NVME_CSTS_RDY); - - // 2. Configure Admin Queue(s) - assert(nvme_asq_region_paddr != 0x0); - assert(nvme_acq_region_paddr != 0x0); - nvme_controller->asq = nvme_asq_region_paddr; - nvme_controller->acq = nvme_acq_region_paddr; - nvme_controller->aqa &= ~(NVME_AQA_ACQS_MASK | NVME_AQA_ASQS_MASK); - nvme_controller->aqa |= ((NVME_ASQ_CAPACITY - 1) << NVME_AQA_ASQS_SHIFT) - | ((NVME_ACQ_CAPACITY - 1) << NVME_AQA_ACQS_SHIFT); - - // 3. Initialise Command Support Sets. - nvme_controller->cc &= ~(NVME_CC_CSS_MASK); - if (nvme_controller->cap & NVME_CAP_NOIOCSS) { - nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; - } else if (nvme_controller->cap & NVME_CAP_IOCSS) { - nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; - } else if (nvme_controller->cap & NVME_CAP_NCSS) { - nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; - } - - // 4a. Arbitration Mechanism (TODO) - // 4b. Memory Page Size - nvme_controller->cc &= ~NVME_CC_MPS_MASK; - /* n.b. page size = 2 ^ (12 + MPS) */ - nvme_controller->cc |= ((PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT) & NVME_CC_MPS_MASK; - - // TODO: See initialisation note under §4.2.4; fine since already that way. - - // 5. Enable the controller - nvme_controller->cc |= NVME_CC_EN; - - // 6. Wait for ready - sddf_dprintf("waiting ready...\n"); - while (!(nvme_controller->csts & NVME_CSTS_RDY)); - sddf_dprintf("\tdone\n"); - - // 7. Send the Identify Controller command (Identify with CNS = 01h) - uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; - uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); - uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); - sddf_dprintf("admin tail: %u\n", sq_tail); - sddf_dprintf("admin head: %u\n", cq_head); - - // Identify Command (5.1.13) - nvme_asq_region[sq_tail] = (nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, - .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, - .dptr_hi = 0, - .dptr_lo = data_region_paddr, /* TEMP */ - }; - - uint32_t orig_dw3 = ((volatile uint32_t *)nvme_acq_region)[3]; - sddf_dprintf("orig dw3: %x\n", orig_dw3); - - // todo: overflow? - nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); - - nvme_debug_dump_controller_regs(); - - // 4.2.4.1 shows how to send commands - sddf_dprintf("waiting for completed command...\n") - while (!(nvme_acq_region[cq_head].phase_tag_and_status & BIT(0))); - sddf_dprintf("complete\n"); - - - // sddf_dprintf("new submission tail: %u\n", nvme_sqytdbl_read(nvme_controller, DSTRD, 0)); - // sddf_dprintf("waiting completion queue change...\n"); - // uint16_t orig_cq_head = cq_head; - // while (cq_head == orig_cq_head) { - // cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); - // } - // sddf_dprintf("\tdone\n"); - - sddf_dprintf("new head: %u\n", cq_head); - // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s - for (int i = 0; i < 64; i++) { - sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); - } -} - -void nvme_init() -{ - sddf_dprintf("Starting NVME config...\n"); - - // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 - - // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 - - nvme_controller_init(); - nvme_queue_init(); -} - void init() { sddf_dprintf("pcie driver starting!\n"); From f58479178e2f053d7a70594411f6ffe0c69fb060 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 30 Sep 2024 17:55:34 +1000 Subject: [PATCH 18/60] weird broken bug Signed-off-by: julia --- drivers/nvme/nvme.c | 50 +++++++++++++++------------------- drivers/nvme/nvme_queue.h | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 29 deletions(-) create mode 100644 drivers/nvme/nvme_queue.h diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 1871c5f2b..37ebb314c 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -1,4 +1,5 @@ #include "nvme.h" +#include "nvme_queue.h" #include @@ -90,45 +91,36 @@ void nvme_controller_init() // 7. Send the Identify Controller command (Identify with CNS = 01h) uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; - uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); - uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); - sddf_dprintf("admin tail: %u\n", sq_tail); - sddf_dprintf("admin head: %u\n", cq_head); // Identify Command (5.1.13) - nvme_asq_region[sq_tail] = (nvme_submission_queue_entry_t){ + nvme_queue_submit(nvme_asq_region, nvme_controller, DSTRD, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, .dptr_hi = 0, .dptr_lo = data_region_paddr, /* TEMP */ - }; - - uint32_t orig_dw3 = ((volatile uint32_t *)nvme_acq_region)[3]; - sddf_dprintf("orig dw3: %x\n", orig_dw3); - - // todo: overflow? - nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); - - nvme_debug_dump_controller_regs(); - - // 4.2.4.1 shows how to send commands - sddf_dprintf("waiting for completed command...\n"); - while (!(nvme_acq_region[cq_head].phase_tag_and_status & BIT(0))); - sddf_dprintf("complete\n"); - + }); + + nvme_completion_queue_entry_t entry; + while (true) { + int ret = nvme_queue_consume(nvme_acq_region, nvme_controller, DSTRD, &entry); + if (ret == 0) { + sddf_dprintf("succeed\n"); + /* succeeded */ + break; + } + sddf_dprintf("fail\n"); + } - // sddf_dprintf("new submission tail: %u\n", nvme_sqytdbl_read(nvme_controller, DSTRD, 0)); - // sddf_dprintf("waiting completion queue change...\n"); - // uint16_t orig_cq_head = cq_head; - // while (cq_head == orig_cq_head) { - // cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); - // } - // sddf_dprintf("\tdone\n"); + sddf_dprintf("CDW0: %04x\n", entry.cdw0); + sddf_dprintf("CDW1: %04x\n", entry.cdw1); + sddf_dprintf("SQHD: %02x\n", entry.sqhd); + sddf_dprintf("SQID: %02x\n", entry.sqid); + sddf_dprintf(" CID: %02x\n", entry.cid); + sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); - sddf_dprintf("new head: %u\n", cq_head); // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s for (int i = 0; i < 64; i++) { - sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); + // sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); } } diff --git a/drivers/nvme/nvme_queue.h b/drivers/nvme/nvme_queue.h new file mode 100644 index 000000000..686af3166 --- /dev/null +++ b/drivers/nvme/nvme_queue.h @@ -0,0 +1,57 @@ +#pragma once + +#include "nvme.h" + +static inline void nvme_queue_submit(volatile nvme_submission_queue_entry_t queue[], volatile void *nvme_controller, + uint16_t DSTRD, nvme_submission_queue_entry_t *entry) +{ + uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); + queue[sq_tail] = *entry; + + // todo: overflow? + nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); +} + + +// TODO: per-queue. +static int phase = 0; + + +// 4.2.41 phase tag +#include +#include +__attribute__((noinline)) +static int nvme_queue_consume(volatile nvme_completion_queue_entry_t queue[], volatile void *nvme_controller, + uint16_t DSTRD, nvme_completion_queue_entry_t *entry) +{ + uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); + + /* if the head is not new */ + THREAD_MEMORY_FENCE(); + if ((queue[cq_head].phase_tag_and_status & BIT(0)) != phase) { + return -1; + } + THREAD_MEMORY_FENCE(); + + // this makes the queue[cq_head] return p (0b1111) + int i = 500; + while (i--) { + seL4_Yield(); + } + + *entry = queue[cq_head]; + microkit_dbg_puts("queue[cq_head]: "); + microkit_dbg_putc(97 + (queue[cq_head].cid & 0b1111)); + microkit_dbg_puts("\nentry: "); + microkit_dbg_putc(97 + (entry->cid & 0b1111)); + microkit_dbg_putc('\n'); + + cq_head++; /* TODO: wrapping */ + // if (cq_head == length) { + // cq_head = 0; + // phase ^= 1; /* flip phase */ + // } + + nvme_cqyhdbl_write(nvme_controller, DSTRD, 0, cq_head); + return 0; +} From 0256cae5225767aed5dfd488c47f93d501eb699a Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 1 Oct 2024 11:47:03 +1000 Subject: [PATCH 19/60] fixed bug :) Signed-off-by: julia --- drivers/nvme/nvme.c | 2 +- drivers/nvme/nvme_queue.h | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 37ebb314c..05174d22b 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -120,7 +120,7 @@ void nvme_controller_init() // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s for (int i = 0; i < 64; i++) { - // sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); + sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); } } diff --git a/drivers/nvme/nvme_queue.h b/drivers/nvme/nvme_queue.h index 686af3166..aa0743858 100644 --- a/drivers/nvme/nvme_queue.h +++ b/drivers/nvme/nvme_queue.h @@ -27,24 +27,11 @@ static int nvme_queue_consume(volatile nvme_completion_queue_entry_t queue[], vo uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); /* if the head is not new */ - THREAD_MEMORY_FENCE(); - if ((queue[cq_head].phase_tag_and_status & BIT(0)) != phase) { + if ((queue[cq_head].phase_tag_and_status & BIT(0)) == phase) { return -1; } - THREAD_MEMORY_FENCE(); - - // this makes the queue[cq_head] return p (0b1111) - int i = 500; - while (i--) { - seL4_Yield(); - } *entry = queue[cq_head]; - microkit_dbg_puts("queue[cq_head]: "); - microkit_dbg_putc(97 + (queue[cq_head].cid & 0b1111)); - microkit_dbg_puts("\nentry: "); - microkit_dbg_putc(97 + (entry->cid & 0b1111)); - microkit_dbg_putc('\n'); cq_head++; /* TODO: wrapping */ // if (cq_head == length) { From ad32403a0871da1aaf7bee19a1b378c6fbea3aae Mon Sep 17 00:00:00 2001 From: julia Date: Wed, 2 Oct 2024 11:47:05 +1000 Subject: [PATCH 20/60] working nvme queue abstraction? Signed-off-by: julia --- drivers/nvme/nvme.c | 29 +++++++------- drivers/nvme/nvme.h | 76 ----------------------------------- drivers/nvme/nvme_queue.h | 83 +++++++++++++++++++++++++++++---------- 3 files changed, 77 insertions(+), 111 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 05174d22b..608638f33 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -4,12 +4,14 @@ #include volatile nvme_controller_t *nvme_controller; -volatile nvme_submission_queue_entry_t *nvme_asq_region; -volatile nvme_completion_queue_entry_t *nvme_acq_region; +nvme_submission_queue_entry_t *nvme_asq_region; +nvme_completion_queue_entry_t *nvme_acq_region; uintptr_t nvme_asq_region_paddr; uintptr_t nvme_acq_region_paddr; #define NVME_ADMIN_QUEUE_SIZE 0x1000 +static nvme_queue_info_t admin_queue_info; +// static nvme_queue_info_t main_io_queue; uintptr_t data_region_paddr; uint8_t *data_region; @@ -20,11 +22,6 @@ uint8_t *data_region; _Static_assert(NVME_ASQ_CAPACITY <= 0x1000, "capacity of ASQ must be <=4096 (entries)"); _Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (entries)"); -/* [NVMe-2.1] 3.3.1 Memory-based Transport Queue Model (PCIe) */ -void nvme_queue_init() -{ - // TODO. -} void nvme_debug_dump_controller_regs() { @@ -54,7 +51,8 @@ void nvme_controller_init() // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) while (nvme_controller->csts & NVME_CSTS_RDY); - // 2. Configure Admin Queue(s) + // 2. Configure Admin Queue(s); i.e. y = 0. + nvme_queues_init(&admin_queue_info, 0, nvme_controller, nvme_asq_region, nvme_acq_region, NVME_ASQ_CAPACITY); // todo: capacity? assert(nvme_asq_region_paddr != 0x0); assert(nvme_acq_region_paddr != 0x0); nvme_controller->asq = nvme_asq_region_paddr; @@ -89,11 +87,8 @@ void nvme_controller_init() while (!(nvme_controller->csts & NVME_CSTS_RDY)); sddf_dprintf("\tdone\n"); - // 7. Send the Identify Controller command (Identify with CNS = 01h) - uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; - - // Identify Command (5.1.13) - nvme_queue_submit(nvme_asq_region, nvme_controller, DSTRD, &(nvme_submission_queue_entry_t){ + // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 + nvme_queue_submit(&admin_queue_info, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, .dptr_hi = 0, @@ -101,14 +96,19 @@ void nvme_controller_init() }); nvme_completion_queue_entry_t entry; + int i = 0; while (true) { - int ret = nvme_queue_consume(nvme_acq_region, nvme_controller, DSTRD, &entry); + int ret = nvme_queue_consume(&admin_queue_info, &entry); if (ret == 0) { sddf_dprintf("succeed\n"); /* succeeded */ break; } sddf_dprintf("fail\n"); + i++; + if (i > 10) { + assert(!"oops"); + } } sddf_dprintf("CDW0: %04x\n", entry.cdw0); @@ -133,5 +133,4 @@ void nvme_init() // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 nvme_controller_init(); - nvme_queue_init(); } diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h index 3f679c3f8..3ff72f23b 100644 --- a/drivers/nvme/nvme.h +++ b/drivers/nvme/nvme.h @@ -131,79 +131,3 @@ _Static_assert(sizeof(nvme_completion_queue_entry_t) == 16, #define NVME_PCIE_CQH_MASK _MASK(0, 15) /* Completion Queue Head */ #define NVME_PCIE_DOORBELL_OFFSET(i, DSTRD) (0x1000 + (i * (4 << DSTRD))) - -/** - * Get the value of the 'Submission Queue y Tail Doorbell' register. - * - * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL - * - * @param nvme_controller Base address of the NVMe controller registers - * @param DSTRD CAP.DSTRD of the NVMe controller - * @param y the submission queue number. the 0th queue is the admin queue. - * - * @todo how big can y be? - */ -static inline uint16_t nvme_sqytdbl_read(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y) -{ - volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y, DSTRD); - return (*addr) & NVME_PCIE_SQT_MASK; -} - -/** - * Set the value of the 'Submission Queue y Tail Doorbell' register. - * - * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL - * - * @param nvme_controller Base address of the NVMe controller registers - * @param DSTRD CAP.DSTRD of the NVMe controller - * @param y the submission queue number. the 0th queue is the admin queue. - * @param tail the new value of the tail register - * - * @todo how big can y be? - */ -static inline void nvme_sqytdbl_write(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y, uint16_t tail) -{ - volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y, DSTRD); - uint32_t val = *addr; - val &= ~NVME_PCIE_SQT_MASK; - val |= tail & NVME_PCIE_SQT_MASK; - *addr = val; -} - -/** - * Get the value of the 'Completion Queue y Head Doorbell' register. - * - * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.2 CQyHDBL - * - * @param nvme_controller Base address of the NVMe controller registers - * @param DSTRD CAP.DSTRD of the NVMe controller - * @param y the completion queue number. the 0th queue is the admin queue. - * - * @todo how big can y be? - */ -static inline uint16_t nvme_cqyhdbl_read(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y) -{ - volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y + 1, DSTRD); - return (*addr) & NVME_PCIE_CQH_MASK; -} - -/** - * Set the value of the 'Completion Queue y Head Doorbell' register. - * - * Ref: [NVMEe-Transport-PCIe-1.1] 3.1.2.2 CQyHDBL - * - * @param nvme_controller Base address of the NVMe controller registers - * @param DSTRD CAP.DSTRD of the NVMe controller - * @param y the submission queue number. the 0th queue is the admin queue. - * @param head the new value of the head register - * - * @todo how big can y be? - */ -static inline void nvme_cqyhdbl_write(volatile void *nvme_controller, uint16_t DSTRD, uint16_t y, uint16_t head) -{ - volatile uint32_t *addr = nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y + 1, DSTRD); - uint32_t val = *addr; - val &= ~NVME_PCIE_CQH_MASK; - val |= head & NVME_PCIE_CQH_MASK; - *addr = val; -} diff --git a/drivers/nvme/nvme_queue.h b/drivers/nvme/nvme_queue.h index aa0743858..651674171 100644 --- a/drivers/nvme/nvme_queue.h +++ b/drivers/nvme/nvme_queue.h @@ -2,43 +2,86 @@ #include "nvme.h" -static inline void nvme_queue_submit(volatile nvme_submission_queue_entry_t queue[], volatile void *nvme_controller, - uint16_t DSTRD, nvme_submission_queue_entry_t *entry) +typedef struct nvme_queue_info { + struct { + nvme_submission_queue_entry_t *queue; + uint16_t tail; + volatile uint32_t *doorbell; + } submission; + + struct { + nvme_completion_queue_entry_t *queue; + uint16_t head; + volatile uint32_t *doorbell; + _Bool phase; + } completion; + + uint16_t capacity; +} nvme_queue_info_t; + + +// y is the submission queue index +static inline void nvme_queues_init(nvme_queue_info_t *queue, uint16_t y, volatile nvme_controller_t *nvme_controller, + nvme_submission_queue_entry_t *submission_queue, + nvme_completion_queue_entry_t *completion_queue, uint16_t capacity) { - uint16_t sq_tail = nvme_sqytdbl_read(nvme_controller, DSTRD, 0); - queue[sq_tail] = *entry; + uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; + + /* [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL */ + volatile uint32_t *submission_doorbell = (void *)nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y, DSTRD); + /* [NVMEe-Transport-PCIe-1.1] 3.1.2.2 CQyHDBL */ + volatile uint32_t *completion_doorbell = (void *)nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y + 1, DSTRD); - // todo: overflow? - nvme_sqytdbl_write(nvme_controller, DSTRD, 0, ++sq_tail); + *queue = (nvme_queue_info_t){ + .submission = { + .queue = submission_queue, + .tail = *submission_doorbell & NVME_PCIE_SQT_MASK, + .doorbell = submission_doorbell, + }, + .completion = { + .queue = completion_queue, + .head = *completion_doorbell & NVME_PCIE_CQH_MASK, + .doorbell = completion_doorbell, + .phase = 0, + }, + .capacity = capacity, + }; } +static inline void nvme_queue_submit(nvme_queue_info_t *queue, nvme_submission_queue_entry_t *entry) +{ + queue->submission.queue[queue->submission.tail] = *entry; -// TODO: per-queue. -static int phase = 0; + // todo: overflow. todo: full queue. + queue->submission.tail++; + uint32_t val = *queue->submission.doorbell; + val &= ~NVME_PCIE_SQT_MASK; + val |= queue->submission.tail & NVME_PCIE_SQT_MASK; + *queue->submission.doorbell = val; +} -// 4.2.41 phase tag -#include -#include -__attribute__((noinline)) -static int nvme_queue_consume(volatile nvme_completion_queue_entry_t queue[], volatile void *nvme_controller, - uint16_t DSTRD, nvme_completion_queue_entry_t *entry) +static inline int nvme_queue_consume(nvme_queue_info_t *queue, nvme_completion_queue_entry_t *entry) { - uint16_t cq_head = nvme_cqyhdbl_read(nvme_controller, DSTRD, 0); - /* if the head is not new */ - if ((queue[cq_head].phase_tag_and_status & BIT(0)) == phase) { + nvme_completion_queue_entry_t *cq_head_entry = &queue->completion.queue[queue->completion.head]; +// 4.2.41 phase tag + if ((cq_head_entry->phase_tag_and_status & BIT(0)) == queue->completion.phase) { return -1; } - *entry = queue[cq_head]; + *entry = *cq_head_entry; - cq_head++; /* TODO: wrapping */ + queue->completion.head++; /* TODO: wrapping */ // if (cq_head == length) { // cq_head = 0; // phase ^= 1; /* flip phase */ // } - nvme_cqyhdbl_write(nvme_controller, DSTRD, 0, cq_head); + uint32_t val = *queue->completion.doorbell; + val &= ~NVME_PCIE_CQH_MASK; + val |= queue->completion.head & NVME_PCIE_CQH_MASK; + *queue->completion.doorbell = val; + return 0; } From d1d5ac5727ee75526cfcb5d1b3d0f3bfd5de7299 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 4 Oct 2024 13:29:24 +1000 Subject: [PATCH 21/60] Properly bracketise macro :) Signed-off-by: julia --- drivers/nvme/nvme.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h index 3ff72f23b..73160fc70 100644 --- a/drivers/nvme/nvme.h +++ b/drivers/nvme/nvme.h @@ -130,4 +130,4 @@ _Static_assert(sizeof(nvme_completion_queue_entry_t) == 16, #define NVME_PCIE_SQT_MASK _MASK(0, 15) /* Submission Queue Tail*/ #define NVME_PCIE_CQH_MASK _MASK(0, 15) /* Completion Queue Head */ -#define NVME_PCIE_DOORBELL_OFFSET(i, DSTRD) (0x1000 + (i * (4 << DSTRD))) +#define NVME_PCIE_DOORBELL_OFFSET(i, DSTRD) (0x1000 + ((i) * (4 << (DSTRD)))) From c8405afe642c3c33d71acf3d08b905da663d9ba9 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 4 Oct 2024 14:46:41 +1000 Subject: [PATCH 22/60] create I/O completion queue Signed-off-by: julia --- drivers/nvme/nvme.c | 75 ++++++++++++++++++++++---- drivers/nvme/nvme.h | 3 -- examples/pcie/board/star64/pcie.system | 10 +++- 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 608638f33..6ec868a75 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -8,20 +8,27 @@ nvme_submission_queue_entry_t *nvme_asq_region; nvme_completion_queue_entry_t *nvme_acq_region; uintptr_t nvme_asq_region_paddr; uintptr_t nvme_acq_region_paddr; +nvme_submission_queue_entry_t *nvme_io_sq_region; +nvme_completion_queue_entry_t *nvme_io_cq_region; +uintptr_t nvme_io_sq_region_paddr; +uintptr_t nvme_io_cq_region_paddr; #define NVME_ADMIN_QUEUE_SIZE 0x1000 +#define NVME_IO_QUEUE_SIZE 0x1000 -static nvme_queue_info_t admin_queue_info; -// static nvme_queue_info_t main_io_queue; +static nvme_queue_info_t admin_queue; +static nvme_queue_info_t io_queue; uintptr_t data_region_paddr; uint8_t *data_region; -/* TODO: don't hardcode */ +/* TODO: don't hardcode 64? is 64 even right for CQ?? */ #define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) #define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) _Static_assert(NVME_ASQ_CAPACITY <= 0x1000, "capacity of ASQ must be <=4096 (entries)"); _Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (entries)"); - +#define NVME_IOQ_CAPACITY (NVME_IO_QUEUE_SIZE / 64) +// §3.3.3.1 +_Static_assert(NVME_ASQ_CAPACITY <= 0x10000, "capacity of ASQ must be <=65536 (slots)"); void nvme_debug_dump_controller_regs() { @@ -52,7 +59,7 @@ void nvme_controller_init() while (nvme_controller->csts & NVME_CSTS_RDY); // 2. Configure Admin Queue(s); i.e. y = 0. - nvme_queues_init(&admin_queue_info, 0, nvme_controller, nvme_asq_region, nvme_acq_region, NVME_ASQ_CAPACITY); // todo: capacity? + nvme_queues_init(&admin_queue, 0, nvme_controller, nvme_asq_region, nvme_acq_region, NVME_ASQ_CAPACITY); // todo: capacity? assert(nvme_asq_region_paddr != 0x0); assert(nvme_acq_region_paddr != 0x0); nvme_controller->asq = nvme_asq_region_paddr; @@ -88,7 +95,9 @@ void nvme_controller_init() sddf_dprintf("\tdone\n"); // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 - nvme_queue_submit(&admin_queue_info, &(nvme_submission_queue_entry_t){ + // TODO: What do we actually need this for???? + // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s + nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, .dptr_hi = 0, @@ -98,7 +107,7 @@ void nvme_controller_init() nvme_completion_queue_entry_t entry; int i = 0; while (true) { - int ret = nvme_queue_consume(&admin_queue_info, &entry); + int ret = nvme_queue_consume(&admin_queue, &entry); if (ret == 0) { sddf_dprintf("succeed\n"); /* succeeded */ @@ -117,11 +126,57 @@ void nvme_controller_init() sddf_dprintf("SQID: %02x\n", entry.sqid); sddf_dprintf(" CID: %02x\n", entry.cid); sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + // 8. The host determines any I/O Command Set specific configuration information + // TODO: Why??? + + // 9. Determine the number of I/O Submission Queues and I/O Completion Queues + // supported using the Set Features command with the Number of Queues feature identifier. + // After determining the number of I/O Queues, the NVMe Transport specific interrupt registers + // (e.g., MSI and/or MSI-X registers) should be configured + // TODO: interrupts. & don't ignore # but we always use one, so. + + // 10. Allocate the appropriate number of I/O Completion Queues based on the + // number required for the system configuration [...]. + // The I/O Completion Queues are allocated using the Create I/O Completion Queue command. + uint16_t io_queue_id = 1; + assert(nvme_io_sq_region != 0x0); + assert(nvme_io_cq_region != 0x0); + assert(nvme_io_sq_region_paddr != 0x0); + assert(nvme_io_cq_region_paddr != 0x0); + nvme_queues_init(&io_queue, io_queue_id, nvme_controller, nvme_io_sq_region, nvme_io_cq_region, NVME_IOQ_CAPACITY); // todo: capacity? + // §5.2.1 + nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, + .cdw10 = /* QSIZE */ (NVME_IOQ_CAPACITY << 16) | /* QID */ io_queue_id, + .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, + .dptr_hi = 0, + .dptr_lo = nvme_io_cq_region_paddr, + }); - // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s - for (int i = 0; i < 64; i++) { - sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); + i = 0; + while (true) { + int ret = nvme_queue_consume(&admin_queue, &entry); + if (ret == 0) { + sddf_dprintf("succeed\n"); + /* succeeded */ + break; + } + sddf_dprintf("fail\n"); + i++; + if (i > 10) { + assert(!"oops"); + } } + + sddf_dprintf("CDW0: %04x\n", entry.cdw0); + sddf_dprintf("CDW1: %04x\n", entry.cdw1); + sddf_dprintf("SQHD: %02x\n", entry.sqhd); + sddf_dprintf("SQID: %02x\n", entry.sqid); + sddf_dprintf(" CID: %02x\n", entry.cid); + sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); } void nvme_init() diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h index 73160fc70..894cf0a02 100644 --- a/drivers/nvme/nvme.h +++ b/drivers/nvme/nvme.h @@ -120,9 +120,6 @@ typedef struct nvme_completion_queue_entry { _Static_assert(sizeof(nvme_completion_queue_entry_t) == 16, "The Common Completion Queue Entry Layout is 16 bytes in size"); - -#define NVME_ADMIN_COMMAND_IDENTIFY 0x6 - /** * Below here is NVMe PCIe Transport Specific Properties. */ diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 5aa41981f..de0565c0f 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -11,8 +11,11 @@ + + + @@ -27,7 +30,12 @@ - + + + + + + From b1494c6cadbc3c08d7388d27bfb63b93023087bc Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 4 Oct 2024 14:57:38 +1000 Subject: [PATCH 23/60] broken i/o submission queeue Signed-off-by: julia --- drivers/nvme/nvme.c | 100 ++++++++++++++++++++++++++++++++++++++++---- drivers/nvme/nvme.h | 14 ++++--- 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 6ec868a75..fa0248eff 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -19,7 +19,7 @@ static nvme_queue_info_t admin_queue; static nvme_queue_info_t io_queue; uintptr_t data_region_paddr; -uint8_t *data_region; +volatile uint8_t *data_region; /* TODO: don't hardcode 64? is 64 even right for CQ?? */ #define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) @@ -136,20 +136,25 @@ void nvme_controller_init() // After determining the number of I/O Queues, the NVMe Transport specific interrupt registers // (e.g., MSI and/or MSI-X registers) should be configured // TODO: interrupts. & don't ignore # but we always use one, so. - - // 10. Allocate the appropriate number of I/O Completion Queues based on the - // number required for the system configuration [...]. - // The I/O Completion Queues are allocated using the Create I/O Completion Queue command. uint16_t io_queue_id = 1; assert(nvme_io_sq_region != 0x0); assert(nvme_io_cq_region != 0x0); assert(nvme_io_sq_region_paddr != 0x0); assert(nvme_io_cq_region_paddr != 0x0); nvme_queues_init(&io_queue, io_queue_id, nvme_controller, nvme_io_sq_region, nvme_io_cq_region, NVME_IOQ_CAPACITY); // todo: capacity? + + // §3.3.1.1 Queue Seutp & Initialization + // => Configures the size of the I/O Submission Queues (CC.IOSQES) and I/O Completion Queues (CC.IOCQES) + // nvme_controller->cc &= ~(NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK); + /* n.b. CQ/SQ entry sizes are specified as 2^n; i.e. 2^4 = 16 and 2^6 = 64. */ + nvme_controller->cc |= (4 << NVME_CC_IOCQES_SHIFT) | (6 << NVME_CC_IOSQES_SHIFT); + + // 10. Allocate the appropriate number of I/O Completion Queues [...] + // The I/O Completion Queues are allocated using the Create I/O Completion Queue command. // §5.2.1 nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, - .cdw10 = /* QSIZE */ (NVME_IOQ_CAPACITY << 16) | /* QID */ io_queue_id, + .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY - 1)<< 16) | /* QID */ io_queue_id, .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, .dptr_hi = 0, .dptr_lo = nvme_io_cq_region_paddr, @@ -176,7 +181,88 @@ void nvme_controller_init() sddf_dprintf("SQID: %02x\n", entry.sqid); sddf_dprintf(" CID: %02x\n", entry.cid); sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + // 11. Allocate the appropriate number of I/O Submission Queues [...] + // The I/O Completion Queues are allocated using the Create I/O Submission Queue command. + // §5.2.2 + sddf_dprintf("sq region paddr: %lx\n", nvme_io_sq_region_paddr); + nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1110 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, + .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1) << 16) | /* QID */ io_queue_id, + .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, + .cdw12 = 0, + .dptr_hi = 0, + .dptr_lo = nvme_io_sq_region_paddr, // TODO: get log info page size byte 0x18, i.e. dptr_lo??? + + // zeroing... + .nsid = 0x0, + .cdw2 = 0x0, + .cdw3 = 0x0, + .mptr = 0x0, + .cdw13 = 0x0, + .cdw14 = 0x0, + .cdw15 = 0x0, + }); + + i = 0; + while (true) { + int ret = nvme_queue_consume(&admin_queue, &entry); + if (ret == 0) { + sddf_dprintf("succeed\n"); + /* succeeded */ + break; + } + sddf_dprintf("fail\n"); + i++; + if (i > 10) { + assert(!"oops"); + } + } + + sddf_dprintf("CDW0: %04x\n", entry.cdw0); + sddf_dprintf("CDW1: %04x\n", entry.cdw1); + sddf_dprintf("SQHD: %02x\n", entry.sqhd); + sddf_dprintf("SQID: %02x\n", entry.sqid); + sddf_dprintf(" CID: %02x\n", entry.cid); + sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); + + // should exist again. + // assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + // try a get log page command (0x2) + nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1001 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .dptr_hi = 0, + .dptr_lo = data_region_paddr, + .cdw10 = /* NUMDL*/ (0x100 << 16) | /* LID*/ 0x01, + .cdw11 = 0x0, + .cdw12 = 0x0, + }); + + i = 0; + while (true) { + int ret = nvme_queue_consume(&admin_queue, &entry); + if (ret == 0) { + sddf_dprintf("succeed\n"); + /* succeeded */ + break; + } + sddf_dprintf("fail\n"); + i++; + if (i > 100) { + assert(!"oops"); + } + } + + sddf_dprintf("CDW0: %04x\n", entry.cdw0); + sddf_dprintf("CDW1: %04x\n", entry.cdw1); + sddf_dprintf("SQHD: %02x\n", entry.sqhd); + sddf_dprintf("SQID: %02x\n", entry.sqid); + sddf_dprintf(" CID: %02x\n", entry.cid); + sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); + for (int i = 0; i < 64; i++) { + sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); + } } void nvme_init() diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h index 894cf0a02..c3e1c1be2 100644 --- a/drivers/nvme/nvme.h +++ b/drivers/nvme/nvme.h @@ -66,11 +66,15 @@ _Static_assert(offsetof(nvme_controller_t, _reserved2) == 0x6C, "nvme_controller #define NVME_CAP_DSTRD_MASK _MASK(32, 35) /* Doorbell Stride (2 ^ (2 + DSTRD)) */ /* [NVMe-2.1] 3.1.4.5 Offset 14h: CC – Controller Configuration */ -#define NVME_CC_MPS_SHIFT 7 /* Host Memory Page Size */ -#define NVME_CC_MPS_MASK _MASK(7, 10) /* Host Memory Page Size */ -#define NVME_CC_CSS_SHIFT 4 /* I/O Command Set Selected */ -#define NVME_CC_CSS_MASK _MASK(4, 6) /* I/O Command Set Selected */ -#define NVME_CC_EN BIT(0) /* Controller Enable */ +#define NVME_CC_IOCQES_SHIFT 20 /* I/O Completion Queue Entry Size */ +#define NVME_CC_IOCQES_MASK _MASK(20, 23) /* I/O Completion Queue Entry Size */ +#define NVME_CC_IOSQES_SHIFT 16 /* I/O Submission Queue Entry Size */ +#define NVME_CC_IOSQES_MASK _MASK(16, 19) /* I/O Submission Queue Entry Size */ +#define NVME_CC_MPS_SHIFT 7 /* Host Memory Page Size */ +#define NVME_CC_MPS_MASK _MASK(7, 10) /* Host Memory Page Size */ +#define NVME_CC_CSS_SHIFT 4 /* I/O Command Set Selected */ +#define NVME_CC_CSS_MASK _MASK(4, 6) /* I/O Command Set Selected */ +#define NVME_CC_EN BIT(0) /* Controller Enable */ /* [NVMe-2.1] 3.1.4.6 Offset 1Ch: CSTS – Controller Status */ #define NVME_CSTS_RDY BIT(0) /* Controller Ready (RO) */ From 676de9c0646dc2a1c188b8476e8c092619a20c1b Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 16:48:44 +1100 Subject: [PATCH 24/60] add poll command helper - still broken i/o submission queue Signed-off-by: julia --- drivers/nvme/nvme.c | 119 ++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 87 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index fa0248eff..362b6d37f 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -30,7 +30,7 @@ _Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (ent // §3.3.3.1 _Static_assert(NVME_ASQ_CAPACITY <= 0x10000, "capacity of ASQ must be <=65536 (slots)"); -void nvme_debug_dump_controller_regs() +static void nvme_debug_dump_controller_regs() { sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, @@ -44,6 +44,26 @@ void nvme_debug_dump_controller_regs() sddf_dprintf("ACQ: %016lx\n", nvme_controller->acq); } +static nvme_completion_queue_entry_t nvme_queue_submit_and_consume_poll(nvme_queue_info_t *queue, + nvme_submission_queue_entry_t *entry) +{ + nvme_queue_submit(queue, entry); + + nvme_completion_queue_entry_t response; + int i = 0; + while (true) { + int ret = nvme_queue_consume(queue, &response); + if (ret == 0) { + sddf_dprintf("received a response for submission with CDW0: %x\n", entry->cdw0); + return response; + } + + if (i % 100 == 0) { + sddf_dprintf("waiting for response to submission with CDW0: %x\n", entry->cdw0); + } + } +} + /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() { @@ -97,35 +117,14 @@ void nvme_controller_init() // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 // TODO: What do we actually need this for???? // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s - nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + nvme_completion_queue_entry_t entry; + entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, .dptr_hi = 0, .dptr_lo = data_region_paddr, /* TEMP */ }); - nvme_completion_queue_entry_t entry; - int i = 0; - while (true) { - int ret = nvme_queue_consume(&admin_queue, &entry); - if (ret == 0) { - sddf_dprintf("succeed\n"); - /* succeeded */ - break; - } - sddf_dprintf("fail\n"); - i++; - if (i > 10) { - assert(!"oops"); - } - } - - sddf_dprintf("CDW0: %04x\n", entry.cdw0); - sddf_dprintf("CDW1: %04x\n", entry.cdw1); - sddf_dprintf("SQHD: %02x\n", entry.sqhd); - sddf_dprintf("SQID: %02x\n", entry.sqid); - sddf_dprintf(" CID: %02x\n", entry.cid); - sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field // 8. The host determines any I/O Command Set specific configuration information @@ -152,7 +151,7 @@ void nvme_controller_init() // 10. Allocate the appropriate number of I/O Completion Queues [...] // The I/O Completion Queues are allocated using the Create I/O Completion Queue command. // §5.2.1 - nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY - 1)<< 16) | /* QID */ io_queue_id, .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, @@ -160,40 +159,21 @@ void nvme_controller_init() .dptr_lo = nvme_io_cq_region_paddr, }); - i = 0; - while (true) { - int ret = nvme_queue_consume(&admin_queue, &entry); - if (ret == 0) { - sddf_dprintf("succeed\n"); - /* succeeded */ - break; - } - sddf_dprintf("fail\n"); - i++; - if (i > 10) { - assert(!"oops"); - } - } - - sddf_dprintf("CDW0: %04x\n", entry.cdw0); - sddf_dprintf("CDW1: %04x\n", entry.cdw1); - sddf_dprintf("SQHD: %02x\n", entry.sqhd); - sddf_dprintf("SQID: %02x\n", entry.sqid); - sddf_dprintf(" CID: %02x\n", entry.cid); - sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field // 11. Allocate the appropriate number of I/O Submission Queues [...] - // The I/O Completion Queues are allocated using the Create I/O Submission Queue command. + // The I/O Submission Queues are allocated using the Create I/O Submission Queue command. // §5.2.2 + sddf_dprintf("!!! IO SQ Create !!!\n"); sddf_dprintf("sq region paddr: %lx\n", nvme_io_sq_region_paddr); - nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1110 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1) << 16) | /* QID */ io_queue_id, .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, .cdw12 = 0, .dptr_hi = 0, - .dptr_lo = nvme_io_sq_region_paddr, // TODO: get log info page size byte 0x18, i.e. dptr_lo??? + // TODO: get log info page size byte 0x18, i.e. dptr_lo??? + .dptr_lo = nvme_io_sq_region_paddr, // zeroing... .nsid = 0x0, @@ -205,32 +185,18 @@ void nvme_controller_init() .cdw15 = 0x0, }); - i = 0; - while (true) { - int ret = nvme_queue_consume(&admin_queue, &entry); - if (ret == 0) { - sddf_dprintf("succeed\n"); - /* succeeded */ - break; - } - sddf_dprintf("fail\n"); - i++; - if (i > 10) { - assert(!"oops"); - } - } - sddf_dprintf("CDW0: %04x\n", entry.cdw0); sddf_dprintf("CDW1: %04x\n", entry.cdw1); sddf_dprintf("SQHD: %02x\n", entry.sqhd); sddf_dprintf("SQID: %02x\n", entry.sqid); sddf_dprintf(" CID: %02x\n", entry.cid); sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); - // should exist again. // assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + // try a get log page command (0x2) - nvme_queue_submit(&admin_queue, &(nvme_submission_queue_entry_t){ + sddf_dprintf("!!! LOG PAGE !!!\n"); + entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1001 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, .dptr_hi = 0, .dptr_lo = data_region_paddr, @@ -239,27 +205,6 @@ void nvme_controller_init() .cdw12 = 0x0, }); - i = 0; - while (true) { - int ret = nvme_queue_consume(&admin_queue, &entry); - if (ret == 0) { - sddf_dprintf("succeed\n"); - /* succeeded */ - break; - } - sddf_dprintf("fail\n"); - i++; - if (i > 100) { - assert(!"oops"); - } - } - - sddf_dprintf("CDW0: %04x\n", entry.cdw0); - sddf_dprintf("CDW1: %04x\n", entry.cdw1); - sddf_dprintf("SQHD: %02x\n", entry.sqhd); - sddf_dprintf("SQID: %02x\n", entry.sqid); - sddf_dprintf(" CID: %02x\n", entry.cid); - sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); for (int i = 0; i < 64; i++) { sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); } From 6d5aca3575b3a1f2153277361cc5230f4a0cdb95 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 17:20:14 +1100 Subject: [PATCH 25/60] figured out the issue... ish... but why?? Signed-off-by: julia --- drivers/nvme/nvme.c | 55 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 362b6d37f..e9b88f2e2 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -64,6 +64,27 @@ static nvme_completion_queue_entry_t nvme_queue_submit_and_consume_poll(nvme_que } } +// TODO: tidy up. +/* 5.1.12.1.2 Error Information (Log Page Identifier 01h) */ +typedef struct { + uint64_t ecnt; /* Error Count; unique ID for this error. (retained across power off) */ + uint16_t sqid; /* Submission Queue ID */ + uint16_t cid; /* Command ID */ + uint16_t sts; /* Status Info (from the completion queue entry - Status + Phase) */ + uint16_t pel; /* Parameter Error location (!!!!) */ + uint64_t lba; /* Logical Block Address */ + uint32_t nsid; + uint8_t vsia; /* vendor specific info available */ + uint8_t trtype; /* transport type */ + uint8_t csi; /* command set indiciator (valid if version > 0x1)*/ + uint8_t opc; /* opcode (valid if version > 0x1)*/ + uint64_t csinfo; /* command specific information (if specified in the command) */ + uint16_t ttsi; /* transport type specification information */ + uint8_t _reserved[20]; + uint8_t lpver; /* log page version */ +} nvme_error_information_log_page_t; +_Static_assert(sizeof(nvme_error_information_log_page_t) == 64, "should be 64 bytes."); + /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() { @@ -172,17 +193,10 @@ void nvme_controller_init() .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, .cdw12 = 0, .dptr_hi = 0, - // TODO: get log info page size byte 0x18, i.e. dptr_lo??? - .dptr_lo = nvme_io_sq_region_paddr, - - // zeroing... - .nsid = 0x0, - .cdw2 = 0x0, - .cdw3 = 0x0, - .mptr = 0x0, - .cdw13 = 0x0, - .cdw14 = 0x0, - .cdw15 = 0x0, + + // Changing sizes in the system file makes different commands work/not. + .dptr_lo = nvme_io_cq_region_paddr, // TODO: This somehow works + // .dptr_lo = nvme_io_sq_region_paddr, // TODO: This doesn't??? }); sddf_dprintf("CDW0: %04x\n", entry.cdw0); @@ -193,6 +207,12 @@ void nvme_controller_init() sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); // should exist again. // assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + if ((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0) { + sddf_dprintf(".... SUCCESS?\n"); + return; + } else { + sddf_dprintf("... FAILED\n"); + } // try a get log page command (0x2) sddf_dprintf("!!! LOG PAGE !!!\n"); @@ -205,8 +225,17 @@ void nvme_controller_init() .cdw12 = 0x0, }); - for (int i = 0; i < 64; i++) { - sddf_dprintf("[%04x]: %x: %c\n", i, data_region[i], data_region[i]); + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + volatile nvme_error_information_log_page_t *errors = (volatile void*)data_region; + for (int i = 0; i < 2; i++) { + sddf_dprintf("Error 0x%lx\n", errors[i].ecnt); + // These produce Store/AMO faults, NEAR THE BEGINNING of this function??? TODO??? + // sddf_dprintf("\tSQID: 0x%x\n", errors[i].sqid); + // sddf_dprintf("\t CID: 0x%x\n", errors[i].cid); + sddf_dprintf("\t STS: 0x%x\n", errors[i].sts); + sddf_dprintf("\t PEL: 0x%x\n", errors[i].pel); + sddf_dprintf("\t(elided)\n"); } } From af9e75f10859f0c96684bd359321d364556a87a6 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 18:13:12 +1100 Subject: [PATCH 26/60] fix MPS; it's a log2-value not just a shifted by 2^12 value turns out I was setting the "host page size" config bits incorrectly, specifying a required page size alignment of 2^13 (i.e. 0x2000, not 0x1000). the first queue was aligned to that, the second was not and was failing. at some point i tried using the same queu eaddress for creating both queues and that worked; so I went "is this some sort of alignment issue?" and went off diving into u-boot to discover it aligns dma buffers to 32bytes which didn't help since they were both page aligned anyway. anyway so i then just started progressively masking off higher and higher bits until the difference between 0x40283000 and 0x40282000 was that one bit and then i was like ??? wtf went scouring through nvme docs, only alignment i could find was dword aligned... then i realised that was mostly talking about PRP entries (scatter-gather list dma) meanwhile i was using just contiguous dma because easier, which references CC.MPS... which i then looked and i thought just doing PAGE_SIZE >> 12 seemed off cause 0x1000 >> 12 = 0x1 and 2^(12 + CC.MPS) is 2^13. and i realised it wanted a log2 value not a shifted value Signed-off-by: julia --- drivers/nvme/nvme.c | 20 ++++++-------------- include/sddf/util/util.h | 5 ----- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index e9b88f2e2..8cf5c3053 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -121,9 +121,11 @@ void nvme_controller_init() // 4a. Arbitration Mechanism (TODO) // 4b. Memory Page Size + // TODO: Check CAP.MPSMAX/CAP.MPSMIN fields nvme_controller->cc &= ~NVME_CC_MPS_MASK; /* n.b. page size = 2 ^ (12 + MPS) */ - nvme_controller->cc |= ((PAGE_SIZE >> 12) << NVME_CC_MPS_SHIFT) & NVME_CC_MPS_MASK; + uint8_t page_size_log2 = 12; /* all architectures we care about have page size 2^12. */ + nvme_controller->cc |= ((page_size_log2 - 12) << NVME_CC_MPS_SHIFT) & NVME_CC_MPS_MASK; // TODO: See initialisation note under §4.2.4; fine since already that way. @@ -185,18 +187,13 @@ void nvme_controller_init() // 11. Allocate the appropriate number of I/O Submission Queues [...] // The I/O Submission Queues are allocated using the Create I/O Submission Queue command. // §5.2.2 - sddf_dprintf("!!! IO SQ Create !!!\n"); - sddf_dprintf("sq region paddr: %lx\n", nvme_io_sq_region_paddr); entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1110 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1) << 16) | /* QID */ io_queue_id, .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, .cdw12 = 0, .dptr_hi = 0, - - // Changing sizes in the system file makes different commands work/not. - .dptr_lo = nvme_io_cq_region_paddr, // TODO: This somehow works - // .dptr_lo = nvme_io_sq_region_paddr, // TODO: This doesn't??? + .dptr_lo = nvme_io_sq_region_paddr, }); sddf_dprintf("CDW0: %04x\n", entry.cdw0); @@ -206,13 +203,8 @@ void nvme_controller_init() sddf_dprintf(" CID: %02x\n", entry.cid); sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); // should exist again. - // assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - if ((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0) { - sddf_dprintf(".... SUCCESS?\n"); - return; - } else { - sddf_dprintf("... FAILED\n"); - } + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + return; // try a get log page command (0x2) sddf_dprintf("!!! LOG PAGE !!!\n"); diff --git a/include/sddf/util/util.h b/include/sddf/util/util.h index aa2f4cdbd..ea743883e 100644 --- a/include/sddf/util/util.h +++ b/include/sddf/util/util.h @@ -67,11 +67,6 @@ void _assert_fail(const char *assertion, const char *file, unsigned int line, #endif #endif -#ifndef PAGE_SIZE -#define PAGE_SIZE_4K 0x1000 -#define PAGE_SIZE PAGE_SIZE_4K -#endif - static inline int sddf_isspace(int ch) { return ch == ' ' || ch == '\f' || ch == '\n' || ch == '\r' || ch == '\t' || ch == '\v'; From 39013cf663ad72bc8f9d5f6b5e08be60cef91df8 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 18:38:24 +1100 Subject: [PATCH 27/60] Move debug code to separate header (for now?) Signed-off-by: julia --- drivers/nvme/nvme.c | 94 +++------------------------------------ drivers/nvme/nvme_debug.h | 94 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 88 deletions(-) create mode 100644 drivers/nvme/nvme_debug.h diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 8cf5c3053..4dde6c4ce 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -1,7 +1,12 @@ +#include + #include "nvme.h" #include "nvme_queue.h" -#include +#define DEBUG_DRIVER +#ifdef DEBUG_DRIVER +#include "nvme_debug.h" +#endif volatile nvme_controller_t *nvme_controller; nvme_submission_queue_entry_t *nvme_asq_region; @@ -30,61 +35,6 @@ _Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (ent // §3.3.3.1 _Static_assert(NVME_ASQ_CAPACITY <= 0x10000, "capacity of ASQ must be <=65536 (slots)"); -static void nvme_debug_dump_controller_regs() -{ - sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); - sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, - nvme_controller->vs.ter); - sddf_dprintf("INTMS: %08x\n", nvme_controller->intms); - sddf_dprintf("INTMC: %08x\n", nvme_controller->intmc); - sddf_dprintf("CC: %08x\n", nvme_controller->cc); - sddf_dprintf("CSTS: %08x\n", nvme_controller->csts); - sddf_dprintf("AQA: %08x\n", nvme_controller->aqa); - sddf_dprintf("ASQ: %016lx\n", nvme_controller->asq); - sddf_dprintf("ACQ: %016lx\n", nvme_controller->acq); -} - -static nvme_completion_queue_entry_t nvme_queue_submit_and_consume_poll(nvme_queue_info_t *queue, - nvme_submission_queue_entry_t *entry) -{ - nvme_queue_submit(queue, entry); - - nvme_completion_queue_entry_t response; - int i = 0; - while (true) { - int ret = nvme_queue_consume(queue, &response); - if (ret == 0) { - sddf_dprintf("received a response for submission with CDW0: %x\n", entry->cdw0); - return response; - } - - if (i % 100 == 0) { - sddf_dprintf("waiting for response to submission with CDW0: %x\n", entry->cdw0); - } - } -} - -// TODO: tidy up. -/* 5.1.12.1.2 Error Information (Log Page Identifier 01h) */ -typedef struct { - uint64_t ecnt; /* Error Count; unique ID for this error. (retained across power off) */ - uint16_t sqid; /* Submission Queue ID */ - uint16_t cid; /* Command ID */ - uint16_t sts; /* Status Info (from the completion queue entry - Status + Phase) */ - uint16_t pel; /* Parameter Error location (!!!!) */ - uint64_t lba; /* Logical Block Address */ - uint32_t nsid; - uint8_t vsia; /* vendor specific info available */ - uint8_t trtype; /* transport type */ - uint8_t csi; /* command set indiciator (valid if version > 0x1)*/ - uint8_t opc; /* opcode (valid if version > 0x1)*/ - uint64_t csinfo; /* command specific information (if specified in the command) */ - uint16_t ttsi; /* transport type specification information */ - uint8_t _reserved[20]; - uint8_t lpver; /* log page version */ -} nvme_error_information_log_page_t; -_Static_assert(sizeof(nvme_error_information_log_page_t) == 64, "should be 64 bytes."); - /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() { @@ -196,39 +146,7 @@ void nvme_controller_init() .dptr_lo = nvme_io_sq_region_paddr, }); - sddf_dprintf("CDW0: %04x\n", entry.cdw0); - sddf_dprintf("CDW1: %04x\n", entry.cdw1); - sddf_dprintf("SQHD: %02x\n", entry.sqhd); - sddf_dprintf("SQID: %02x\n", entry.sqid); - sddf_dprintf(" CID: %02x\n", entry.cid); - sddf_dprintf("P&STATUS: %02x\n", entry.phase_tag_and_status); - // should exist again. assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - return; - - // try a get log page command (0x2) - sddf_dprintf("!!! LOG PAGE !!!\n"); - entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1001 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, - .dptr_hi = 0, - .dptr_lo = data_region_paddr, - .cdw10 = /* NUMDL*/ (0x100 << 16) | /* LID*/ 0x01, - .cdw11 = 0x0, - .cdw12 = 0x0, - }); - - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - - volatile nvme_error_information_log_page_t *errors = (volatile void*)data_region; - for (int i = 0; i < 2; i++) { - sddf_dprintf("Error 0x%lx\n", errors[i].ecnt); - // These produce Store/AMO faults, NEAR THE BEGINNING of this function??? TODO??? - // sddf_dprintf("\tSQID: 0x%x\n", errors[i].sqid); - // sddf_dprintf("\t CID: 0x%x\n", errors[i].cid); - sddf_dprintf("\t STS: 0x%x\n", errors[i].sts); - sddf_dprintf("\t PEL: 0x%x\n", errors[i].pel); - sddf_dprintf("\t(elided)\n"); - } } void nvme_init() diff --git a/drivers/nvme/nvme_debug.h b/drivers/nvme/nvme_debug.h new file mode 100644 index 000000000..5c7892dcf --- /dev/null +++ b/drivers/nvme/nvme_debug.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include "nvme.h" +#include "nvme_queue.h" +#include + +// clang-format off +#define LOG_NVME(...) do{ sddf_dprintf("NVME|INFO: "); sddf_dprintf(__VA_ARGS__); }while(0) +// clang-format on + +static void nvme_debug_dump_controller_regs(nvme_controller_t *nvme_controller) +{ + LOG_NVME("CAP: %016lx\n", nvme_controller->cap); + LOG_NVME("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + nvme_controller->vs.ter); + LOG_NVME("INTMS: %08x\n", nvme_controller->intms); + LOG_NVME("INTMC: %08x\n", nvme_controller->intmc); + LOG_NVME("CC: %08x\n", nvme_controller->cc); + LOG_NVME("CSTS: %08x\n", nvme_controller->csts); + LOG_NVME("AQA: %08x\n", nvme_controller->aqa); + LOG_NVME("ASQ: %016lx\n", nvme_controller->asq); + LOG_NVME("ACQ: %016lx\n", nvme_controller->acq); +} + +static nvme_completion_queue_entry_t nvme_queue_submit_and_consume_poll(nvme_queue_info_t *queue, + nvme_submission_queue_entry_t *entry) +{ + nvme_queue_submit(queue, entry); + + nvme_completion_queue_entry_t response; + int i = 0; + while (true) { + int ret = nvme_queue_consume(queue, &response); + if (ret == 0) { + LOG_NVME("received a response for submission with CDW0: %x\n", entry->cdw0); + return response; + } + + if (i % 100 == 0) { + LOG_NVME("waiting for response to submission with CDW0: %x\n", entry->cdw0); + } + } +} + +// TODO: tidy up. +/* 5.1.12.1.2 Error Information (Log Page Identifier 01h) */ +typedef struct { + uint64_t ecnt; /* Error Count; unique ID for this error. (retained across power off) */ + uint16_t sqid; /* Submission Queue ID */ + uint16_t cid; /* Command ID */ + uint16_t sts; /* Status Info (from the completion queue entry - Status + Phase) */ + uint16_t pel; /* Parameter Error location (!!!!) */ + uint64_t lba; /* Logical Block Address */ + uint32_t nsid; + uint8_t vsia; /* vendor specific info available */ + uint8_t trtype; /* transport type */ + uint8_t csi; /* command set indiciator (valid if version > 0x1)*/ + uint8_t opc; /* opcode (valid if version > 0x1)*/ + uint64_t csinfo; /* command specific information (if specified in the command) */ + uint16_t ttsi; /* transport type specification information */ + uint8_t _reserved[20]; + uint8_t lpver; /* log page version */ +} nvme_error_information_log_page_t; +_Static_assert(sizeof(nvme_error_information_log_page_t) == 64, "should be 64 bytes."); + +static void nvme_debug_get_error_information_log_page(nvme_queue_info_t *admin_queue, uint64_t data_paddr, + volatile void *data) +{ + LOG_NVME("!!! LOG PAGE !!!\n"); + nvme_completion_queue_entry_t entry; + entry = nvme_queue_submit_and_consume_poll( + admin_queue, &(nvme_submission_queue_entry_t) { + .cdw0 = /* CID */ (0b1001 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .dptr_hi = 0, + .dptr_lo = data_paddr, + .cdw10 = /* NUMDL*/ (0x100 << 16) | /* LID*/ 0x01, + .cdw11 = 0x0, + .cdw12 = 0x0, + }); + + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + volatile nvme_error_information_log_page_t *errors = data; + for (int i = 0; i < 2; i++) { + LOG_NVME("Error 0x%lx\n", errors[i].ecnt); + // These produce Store/AMO faults, NEAR THE BEGINNING of this function??? TODO??? + // LOG_NVME("\tSQID: 0x%x\n", errors[i].sqid); + // LOG_NVME("\t CID: 0x%x\n", errors[i].cid); + LOG_NVME("\t STS: 0x%x\n", errors[i].sts); + LOG_NVME("\t PEL: 0x%x\n", errors[i].pel); + LOG_NVME("\t(elided)\n"); + } +} From 64edacd670f9923b79230e6f7a195c6ab3e679d3 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 18:48:20 +1100 Subject: [PATCH 28/60] yes this is a whole another specification https://metebalci.com/blog/a-quick-tour-of-nvm-express-nvme/ Signed-off-by: julia --- drivers/nvme/nvme.c | 6 ++++++ drivers/nvme/nvme.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 4dde6c4ce..903b7aee7 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -147,6 +147,12 @@ void nvme_controller_init() }); assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + // 12. To enable asynchronous notification of optional events, the host should issue a Set Features + // command specifying the events to enable. To enable asynchronous notification of events, the host + // should submit an appropriate number of Asynchronous Event Request commands. This step may + // be done at any point after the controller signals that the controller is ready (i.e., CSTS.RDY is set to ‘1’). + // TODO: ??? } void nvme_init() diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h index c3e1c1be2..300f33b2a 100644 --- a/drivers/nvme/nvme.h +++ b/drivers/nvme/nvme.h @@ -13,6 +13,9 @@ [NVMEe-Transport-PCIe-1.1] NVMe over PCIe Transport Specification, Revision 1.1 (Aug 5, 2024) https://nvmexpress.org/wp-content/uploads/NVM-Express-PCI-Express-Transport-Specification-Revision-1.1-2024.08.05-Ratified.pdf + + [NVMe-CommandSet-1.1] NVM Command Set Specification, Revision 1.1 (Aug 5, 2024) + https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-Revision-1.1-2024.08.05-Ratified.pdf */ /* [NVMe-2.1] Section 3.1.4 Controller Properties From 495417c4dcb8d2acdac79abdd7a32b759c26de06 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 20:05:29 +1100 Subject: [PATCH 29/60] try to read data this traps with unimp :( Signed-off-by: julia --- drivers/nvme/nvme.c | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 903b7aee7..3f6066ffa 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -38,11 +38,10 @@ _Static_assert(NVME_ASQ_CAPACITY <= 0x10000, "capacity of ASQ must be <=65536 (s /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() { - sddf_dprintf("CAP: %016lx\n", nvme_controller->cap); - sddf_dprintf("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, - nvme_controller->vs.ter); - sddf_dprintf("CC: %08x\n", nvme_controller->cc); - + LOG_NVME("CAP: %016lx\n", nvme_controller->cap); + LOG_NVME("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + nvme_controller->vs.ter); + LOG_NVME("CC: %08x\n", nvme_controller->cc); nvme_controller->cc &= ~NVME_CC_EN; @@ -83,9 +82,9 @@ void nvme_controller_init() nvme_controller->cc |= NVME_CC_EN; // 6. Wait for ready - sddf_dprintf("waiting ready...\n"); + LOG_NVME("waiting ready...\n"); while (!(nvme_controller->csts & NVME_CSTS_RDY)); - sddf_dprintf("\tdone\n"); + LOG_NVME("\tdone\n"); // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 // TODO: What do we actually need this for???? @@ -126,7 +125,7 @@ void nvme_controller_init() // §5.2.1 entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, - .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY - 1)<< 16) | /* QID */ io_queue_id, + .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY - 1U) << 16) | /* QID */ io_queue_id, .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, .dptr_hi = 0, .dptr_lo = nvme_io_cq_region_paddr, @@ -139,7 +138,7 @@ void nvme_controller_init() // §5.2.2 entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1110 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, - .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1) << 16) | /* QID */ io_queue_id, + .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1U) << 16) | /* QID */ io_queue_id, .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, .cdw12 = 0, .dptr_hi = 0, @@ -157,11 +156,32 @@ void nvme_controller_init() void nvme_init() { - sddf_dprintf("Starting NVME config...\n"); + LOG_NVME("Starting NVME config... (%s)\n", microkit_name); // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 nvme_controller_init(); + + + /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ + nvme_completion_queue_entry_t entry; + uint16_t number_blocks = 10; + entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1 << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), + .dptr_hi = 0x0, + .dptr_lo = data_region_paddr, + }); + + LOG_NVME("CDW0: %04x\n", entry.cdw0); + LOG_NVME("CDW1: %04x\n", entry.cdw1); + LOG_NVME("SQHD: %02x\n", entry.sqhd); + LOG_NVME("SQID: %02x\n", entry.sqid); + LOG_NVME(" CID: %02x\n", entry.cid); + LOG_NVME("P&STATUS: %02x\n", entry.phase_tag_and_status); + } From 3834263cd9907099af0f5ae9aa6631e6752db29f Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 7 Oct 2024 20:08:06 +1100 Subject: [PATCH 30/60] i spent way too long trying to figure out why this was trapping... .. or being entirely optimised out https://cerberus.cl.cam.ac.uk/?short/cea8cb this is undefined behaviour. god I love C Signed-off-by: julia --- drivers/nvme/nvme.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 3f6066ffa..84dc2a05c 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -172,7 +172,7 @@ void nvme_init() .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, .cdw10 = /* SLBA[31:00] */ 0x0, .cdw11 = /* SLBA[63:32] */ 0x0, - .cdw12 = /* LR */ (0b1 << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), .dptr_hi = 0x0, .dptr_lo = data_region_paddr, }); From 83039ab11a02ff5abb70959b28978e98fb7a59c9 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 8 Oct 2024 11:28:24 +1100 Subject: [PATCH 31/60] use PRP entries as this is w hat they are Signed-off-by: julia --- drivers/nvme/nvme.c | 12 ++++++------ drivers/nvme/nvme.h | 14 +++++++------- drivers/nvme/nvme_debug.h | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 84dc2a05c..d626800c8 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -93,8 +93,8 @@ void nvme_controller_init() entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1111 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x6, .cdw10 = /* CNTID[31:16] */ 0x0 | /* CNS */ 0x01, - .dptr_hi = 0, - .dptr_lo = data_region_paddr, /* TEMP */ + .prp2 = 0, + .prp1 = data_region_paddr, /* TEMP */ }); assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field @@ -127,8 +127,8 @@ void nvme_controller_init() .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY - 1U) << 16) | /* QID */ io_queue_id, .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, - .dptr_hi = 0, - .dptr_lo = nvme_io_cq_region_paddr, + .prp2 = 0, + .prp1 = nvme_io_cq_region_paddr, }); assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field @@ -141,8 +141,8 @@ void nvme_controller_init() .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1U) << 16) | /* QID */ io_queue_id, .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, .cdw12 = 0, - .dptr_hi = 0, - .dptr_lo = nvme_io_sq_region_paddr, + .prp2 = 0, + .prp1 = nvme_io_sq_region_paddr, }); assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field diff --git a/drivers/nvme/nvme.h b/drivers/nvme/nvme.h index 300f33b2a..1a1f5618f 100644 --- a/drivers/nvme/nvme.h +++ b/drivers/nvme/nvme.h @@ -99,13 +99,13 @@ _Static_assert(offsetof(nvme_controller_t, _reserved2) == 0x6C, "nvme_controller typedef struct nvme_submission_queue_entry { // TODO: split out to opcode etc (figure 91??) - uint32_t cdw0; /* Command Dword 0 (common) */ - uint32_t nsid; /* Namespace Identifier */ - uint32_t cdw2; /* Command Dword 2 (command-specific) */ - uint32_t cdw3; /* Command Dword 3 (command-specific) */ - uint64_t mptr; /* Metadata Pointer */ - uint64_t dptr_lo; /* Data Pointer (low 8 bytes) */ - uint64_t dptr_hi; /* Data Pointer (high 8 bytes) */ + uint32_t cdw0; /* Command Dword 0 (common) */ + uint32_t nsid; /* Namespace Identifier */ + uint32_t cdw2; /* Command Dword 2 (command-specific) */ + uint32_t cdw3; /* Command Dword 3 (command-specific) */ + uint64_t mptr; /* Metadata Pointer */ + uint64_t prp1; /* Data Pointer - PRP Entry 1 */ + uint64_t prp2; /* Data Pointer - PRP Entry 2 */ uint32_t cdw10; /* Command Dword 10 (command-specific) */ uint32_t cdw11; /* Command Dword 11 (command-specific) */ uint32_t cdw12; /* Command Dword 12 (command-specific) */ diff --git a/drivers/nvme/nvme_debug.h b/drivers/nvme/nvme_debug.h index 5c7892dcf..b017294ae 100644 --- a/drivers/nvme/nvme_debug.h +++ b/drivers/nvme/nvme_debug.h @@ -72,8 +72,8 @@ static void nvme_debug_get_error_information_log_page(nvme_queue_info_t *admin_q entry = nvme_queue_submit_and_consume_poll( admin_queue, &(nvme_submission_queue_entry_t) { .cdw0 = /* CID */ (0b1001 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, - .dptr_hi = 0, - .dptr_lo = data_paddr, + .prp2 = 0, + .prp1 = data_paddr, .cdw10 = /* NUMDL*/ (0x100 << 16) | /* LID*/ 0x01, .cdw11 = 0x0, .cdw12 = 0x0, From d477eb9e2f63b97b7d4508b9b5800a3588971731 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 8 Oct 2024 11:42:41 +1100 Subject: [PATCH 32/60] it works!!! Signed-off-by: julia --- drivers/nvme/nvme.c | 21 ++++++++++++--------- drivers/nvme/nvme_debug.h | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index d626800c8..6b72b1f06 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -164,24 +164,27 @@ void nvme_init() nvme_controller_init(); + sddf_memset((void *)data_region, 'a', 512); + for (int i = 0; i < 10; i++) { + sddf_dprintf("Data [%02x]: %04x\n", i, data_region[i]); + } /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ nvme_completion_queue_entry_t entry; - uint16_t number_blocks = 10; + uint16_t number_blocks = 1; entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? .cdw10 = /* SLBA[31:00] */ 0x0, .cdw11 = /* SLBA[63:32] */ 0x0, .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), - .dptr_hi = 0x0, - .dptr_lo = data_region_paddr, + .prp2 = 0x0, + .prp1 = data_region_paddr, }); - LOG_NVME("CDW0: %04x\n", entry.cdw0); - LOG_NVME("CDW1: %04x\n", entry.cdw1); - LOG_NVME("SQHD: %02x\n", entry.sqhd); - LOG_NVME("SQID: %02x\n", entry.sqid); - LOG_NVME(" CID: %02x\n", entry.cid); - LOG_NVME("P&STATUS: %02x\n", entry.phase_tag_and_status); + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + for (int i = 0; i < 10; i++) { + sddf_dprintf("Data [%02x]: %04x\n", i, data_region[i]); + } } diff --git a/drivers/nvme/nvme_debug.h b/drivers/nvme/nvme_debug.h index b017294ae..dc3e69911 100644 --- a/drivers/nvme/nvme_debug.h +++ b/drivers/nvme/nvme_debug.h @@ -85,8 +85,8 @@ static void nvme_debug_get_error_information_log_page(nvme_queue_info_t *admin_q for (int i = 0; i < 2; i++) { LOG_NVME("Error 0x%lx\n", errors[i].ecnt); // These produce Store/AMO faults, NEAR THE BEGINNING of this function??? TODO??? - // LOG_NVME("\tSQID: 0x%x\n", errors[i].sqid); - // LOG_NVME("\t CID: 0x%x\n", errors[i].cid); + LOG_NVME("\tSQID: 0x%x\n", errors[i].sqid); + LOG_NVME("\t CID: 0x%x\n", errors[i].cid); LOG_NVME("\t STS: 0x%x\n", errors[i].sts); LOG_NVME("\t PEL: 0x%x\n", errors[i].pel); LOG_NVME("\t(elided)\n"); From 153848d117b454b8a4f6b86a73b082afab2569c4 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 8 Oct 2024 14:59:12 +1100 Subject: [PATCH 33/60] fix queue lengths for I/O Signed-off-by: julia --- drivers/nvme/nvme.c | 28 +++++++++++++--------------- drivers/nvme/nvme_queue.h | 10 ++++++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 6b72b1f06..2abd07133 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -26,14 +26,15 @@ static nvme_queue_info_t io_queue; uintptr_t data_region_paddr; volatile uint8_t *data_region; -/* TODO: don't hardcode 64? is 64 even right for CQ?? */ -#define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) -#define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / 64) +#define NVME_ASQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / sizeof(nvme_submission_queue_entry_t)) +#define NVME_ACQ_CAPACITY (NVME_ADMIN_QUEUE_SIZE / sizeof(nvme_completion_queue_entry_t)) _Static_assert(NVME_ASQ_CAPACITY <= 0x1000, "capacity of ASQ must be <=4096 (entries)"); _Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (entries)"); -#define NVME_IOQ_CAPACITY (NVME_IO_QUEUE_SIZE / 64) +#define NVME_IO_SQ_CAPACITY (NVME_IO_QUEUE_SIZE / sizeof(nvme_submission_queue_entry_t)) +#define NVME_IO_CQ_CAPACITY (NVME_IO_QUEUE_SIZE / sizeof(nvme_completion_queue_entry_t)) // §3.3.3.1 -_Static_assert(NVME_ASQ_CAPACITY <= 0x10000, "capacity of ASQ must be <=65536 (slots)"); +_Static_assert(NVME_IO_SQ_CAPACITY <= 0x10000, "capacity of IO SQ must be <=65536 (entries)"); +_Static_assert(NVME_IO_CQ_CAPACITY <= 0x10000, "capacity of IO CQ must be <=65536 (entries)"); /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() @@ -48,8 +49,9 @@ void nvme_controller_init() // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) while (nvme_controller->csts & NVME_CSTS_RDY); - // 2. Configure Admin Queue(s); i.e. y = 0. - nvme_queues_init(&admin_queue, 0, nvme_controller, nvme_asq_region, nvme_acq_region, NVME_ASQ_CAPACITY); // todo: capacity? + // 2. Configure Admin Queue(s); + nvme_queues_init(&admin_queue, /* y */ 0, nvme_controller, nvme_asq_region, NVME_ASQ_CAPACITY, nvme_acq_region, + NVME_ACQ_CAPACITY); assert(nvme_asq_region_paddr != 0x0); assert(nvme_acq_region_paddr != 0x0); nvme_controller->asq = nvme_asq_region_paddr; @@ -112,7 +114,8 @@ void nvme_controller_init() assert(nvme_io_cq_region != 0x0); assert(nvme_io_sq_region_paddr != 0x0); assert(nvme_io_cq_region_paddr != 0x0); - nvme_queues_init(&io_queue, io_queue_id, nvme_controller, nvme_io_sq_region, nvme_io_cq_region, NVME_IOQ_CAPACITY); // todo: capacity? + nvme_queues_init(&io_queue, io_queue_id, nvme_controller, nvme_io_sq_region, NVME_IO_SQ_CAPACITY, nvme_io_cq_region, + NVME_IO_CQ_CAPACITY); // §3.3.1.1 Queue Seutp & Initialization // => Configures the size of the I/O Submission Queues (CC.IOSQES) and I/O Completion Queues (CC.IOCQES) @@ -125,7 +128,7 @@ void nvme_controller_init() // §5.2.1 entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, - .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY - 1U) << 16) | /* QID */ io_queue_id, + .cdw10 = /* QSIZE */ ((NVME_IO_CQ_CAPACITY - 1U) << 16) | /* QID */ io_queue_id, .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, .prp2 = 0, .prp1 = nvme_io_cq_region_paddr, @@ -138,7 +141,7 @@ void nvme_controller_init() // §5.2.2 entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1110 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, - .cdw10 = /* QSIZE */ ((NVME_IOQ_CAPACITY-1U) << 16) | /* QID */ io_queue_id, + .cdw10 = /* QSIZE */ ((NVME_IO_SQ_CAPACITY - 1U) << 16) | /* QID */ io_queue_id, .cdw11 = /* CQID */ (io_queue_id << 16) | /* QPRIO */ (0b00 << 1) | /* PC */ 0b1, .cdw12 = 0, .prp2 = 0, @@ -164,11 +167,6 @@ void nvme_init() nvme_controller_init(); - sddf_memset((void *)data_region, 'a', 512); - for (int i = 0; i < 10; i++) { - sddf_dprintf("Data [%02x]: %04x\n", i, data_region[i]); - } - /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ nvme_completion_queue_entry_t entry; uint16_t number_blocks = 1; diff --git a/drivers/nvme/nvme_queue.h b/drivers/nvme/nvme_queue.h index 651674171..dbd10570c 100644 --- a/drivers/nvme/nvme_queue.h +++ b/drivers/nvme/nvme_queue.h @@ -5,25 +5,26 @@ typedef struct nvme_queue_info { struct { nvme_submission_queue_entry_t *queue; + uint16_t capacity; uint16_t tail; volatile uint32_t *doorbell; } submission; struct { nvme_completion_queue_entry_t *queue; + uint16_t capacity; uint16_t head; volatile uint32_t *doorbell; _Bool phase; } completion; - uint16_t capacity; } nvme_queue_info_t; // y is the submission queue index static inline void nvme_queues_init(nvme_queue_info_t *queue, uint16_t y, volatile nvme_controller_t *nvme_controller, - nvme_submission_queue_entry_t *submission_queue, - nvme_completion_queue_entry_t *completion_queue, uint16_t capacity) + nvme_submission_queue_entry_t *submission_queue, uint16_t submission_capacity, + nvme_completion_queue_entry_t *completion_queue, uint16_t completion_capacity) { uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; @@ -35,16 +36,17 @@ static inline void nvme_queues_init(nvme_queue_info_t *queue, uint16_t y, volati *queue = (nvme_queue_info_t){ .submission = { .queue = submission_queue, + .capacity = submission_capacity, .tail = *submission_doorbell & NVME_PCIE_SQT_MASK, .doorbell = submission_doorbell, }, .completion = { .queue = completion_queue, + .capacity = completion_capacity, .head = *completion_doorbell & NVME_PCIE_CQH_MASK, .doorbell = completion_doorbell, .phase = 0, }, - .capacity = capacity, }; } From 56b86e3d07eeedeba6a984932f864772b9b3340a Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 8 Oct 2024 15:26:05 +1100 Subject: [PATCH 34/60] read & write works Signed-off-by: julia --- drivers/nvme/nvme.c | 51 ++++++++++++++++++-------- examples/pcie/board/star64/pcie.system | 2 +- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 2abd07133..bb11a8e0e 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -167,22 +167,41 @@ void nvme_init() nvme_controller_init(); - /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ - nvme_completion_queue_entry_t entry; - uint16_t number_blocks = 1; - entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, - .nsid = 0x1, // TOOD: Why is NSID 1 now ???? - .cdw10 = /* SLBA[31:00] */ 0x0, - .cdw11 = /* SLBA[63:32] */ 0x0, - .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), - .prp2 = 0x0, - .prp1 = data_region_paddr, - }); - - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + for (int z = 0; z < 9; z++) { + /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ + nvme_completion_queue_entry_t entry; + uint16_t number_blocks = 1; + entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), + .prp2 = 0x0, + .prp1 = data_region_paddr, + }); + + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + for (int i = 0; i < 32; i++) { + sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); + } + + for (int i = 0; i < 4096; i++) { + data_region[i] = data_region[i] ^ 0xbb; + } + + entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1101 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), + .prp2 = 0x0, + .prp1 = data_region_paddr, + }); + + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - for (int i = 0; i < 10; i++) { - sddf_dprintf("Data [%02x]: %04x\n", i, data_region[i]); } } diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index de0565c0f..e595adf18 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -19,7 +19,7 @@ - + From 7de9e928c38e98b2fe6b78f5468b7140437c74f3 Mon Sep 17 00:00:00 2001 From: julia Date: Wed, 9 Oct 2024 11:42:18 +1100 Subject: [PATCH 35/60] Get a PCIe interrupt Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 5 ++--- examples/pcie/board/star64/pcie.system | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 9b93ff136..00efb2ef1 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -168,9 +168,8 @@ void init() nvme_init(); } -/* See: 3.3 NVM Queue MOdels */ -// CAP.MQES - void notified(microkit_channel ch) { + sddf_dprintf("notified on ch: %u\n", ch); + microkit_irq_ack(ch); } diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index e595adf18..37c7e580f 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -22,6 +22,7 @@ + From 012c50121bf93d817991ac131d588567510c6e48 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 14 Oct 2024 11:33:29 +1100 Subject: [PATCH 36/60] debug register fix Signed-off-by: julia --- drivers/nvme/nvme_debug.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/nvme_debug.h b/drivers/nvme/nvme_debug.h index dc3e69911..c0c1e03f0 100644 --- a/drivers/nvme/nvme_debug.h +++ b/drivers/nvme/nvme_debug.h @@ -9,7 +9,7 @@ #define LOG_NVME(...) do{ sddf_dprintf("NVME|INFO: "); sddf_dprintf(__VA_ARGS__); }while(0) // clang-format on -static void nvme_debug_dump_controller_regs(nvme_controller_t *nvme_controller) +static void nvme_debug_dump_controller_regs(volatile nvme_controller_t *nvme_controller) { LOG_NVME("CAP: %016lx\n", nvme_controller->cap); LOG_NVME("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, From 4f4ae04d489c2df61afc8ed1bea5621657817ab0 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 14 Oct 2024 17:07:45 +1100 Subject: [PATCH 37/60] some extra asserts Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 4 +++- examples/pcie/board/star64/pcie.system | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 00efb2ef1..46777ef2f 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -38,7 +38,9 @@ uintptr_t get_bdf_offset(uint8_t bus, uint8_t device, uint8_t function) void device_print(uint8_t bus, uint8_t device, uint8_t function) { - void *config_base = (void *)(pcie_config + get_bdf_offset(bus, device, function)); + uintptr_t offset = get_bdf_offset(bus, device, function); + assert(offset < PCIE_CONFIG_SIZE); + void *config_base = (void *)(pcie_config + offset); volatile pcie_header_t *header = (pcie_header_t *)config_base; if (header->vendor_id == PCIE_VENDOR_INVALID) { diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 37c7e580f..8f409d136 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -22,7 +22,7 @@ - + From df9eab7a4a422c8de3c8f8207dfe26c034b7e41b Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 11:44:55 +1100 Subject: [PATCH 38/60] dodgy stuff Signed-off-by: julia --- drivers/nvme/nvme.c | 15 ++++-- drivers/pcie/starfive/pcie.c | 90 +++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index bb11a8e0e..ddf8ff241 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -88,6 +88,10 @@ void nvme_controller_init() while (!(nvme_controller->csts & NVME_CSTS_RDY)); LOG_NVME("\tdone\n"); + /* https://github.com/redox-os/drivers/blob/master/storage/nvmed/src/nvme/mod.rs#L292*/ + nvme_controller->intms = 0xFFFFFFFF; + nvme_controller->intmc = 0x00000001; + // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 // TODO: What do we actually need this for???? // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s @@ -101,6 +105,10 @@ void nvme_controller_init() assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + nvme_controller->intms = 0x00000001; + /* At this point: we start getting interrupts */ + return; + // 8. The host determines any I/O Command Set specific configuration information // TODO: Why??? @@ -166,6 +174,7 @@ void nvme_init() // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 nvme_controller_init(); + return; for (int z = 0; z < 9; z++) { /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ @@ -183,9 +192,9 @@ void nvme_init() assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - for (int i = 0; i < 32; i++) { - sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); - } + // for (int i = 0; i < 32; i++) { + // sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); + // } for (int i = 0; i < 4096; i++) { data_region[i] = data_region[i] ^ 0xbb; diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 46777ef2f..56a7af5e4 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -147,10 +147,81 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) } } +void device_ack_irq(uint8_t bus, uint8_t device, uint8_t function, bool to_mask) +{ + uintptr_t offset = get_bdf_offset(bus, device, function); + assert(offset < PCIE_CONFIG_SIZE); + void *config_base = (void *)(pcie_config + offset); + volatile pcie_header_t *header = (pcie_header_t *)config_base; + + if (header->vendor_id == PCIE_VENDOR_INVALID) return; + + sddf_dprintf("\nB.D:F: %02x:%02x.%01x\n", bus, device, function); + sddf_dprintf("ACK: command register: 0x%04x\n", header->command); + sddf_dprintf("ACK: status register: 0x%04x\n", header->status); + sddf_dprintf("\t(PIN-based) interrupt disable: %s\n", (header->command & BIT(10)) ? "is disabled" : "is not disabled"); + sddf_dprintf("\t(PIN-based) interrupt status: %s\n", (header->status & BIT(3)) ? "asserted" : "none"); + + /* mask out the interrupt... */ + // https://elixir.bootlin.com/linux/v5.18-rc4/source/drivers/pci/pci.c#L4595 + if (to_mask) { + header->command |= BIT(10); + } else { + header->command &= ~BIT(10); + } + + // we only access IRQ PIn and Line an Capiblites which are actually comon (TODO) + // assert((header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK) == PCIE_HEADER_TYPE_GENERAL); + volatile pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; + // 0xff (255) means "unknown" or "no connection" on interrupt line + // interrupt pin: 0 = no, 1 => INTA#, 2 => INTB#, 3 => INTC#, 4 => INTD# + sddf_printf("\t(PIN-based) interrupt line: %02x, interrupt pin: %02x\n", type0_header->interrupt_line, type0_header->interrupt_pin); + + assert(header->status & BIT(4)); /* assert capabilities list status exists */ + sddf_dprintf("start capabilities PTR: %u\n", type0_header->capabilities_pointer); + + /* ach capability +must be DWORD aligned. The bottom two bits of all pointers (including the initial pointer +at 34h) are reserved and must be implemented as 00b although software must mask them to +allow for future uses of these bits. A pointer value of 00h is used to indicate the last +capability in the list.*/ + uint8_t current_ptr = type0_header->capabilities_pointer & ~0b11; + /* process capabilities list to get MSI */ + while (current_ptr != 0) { + volatile uint8_t *capability = (void *)(config_base + current_ptr); + /* 6.8.1 of the pci spec */ + uint8_t cap_id = capability[0]; + sddf_dprintf("cap ptr: %x, id: %x\n", current_ptr, cap_id); + uint8_t PCIE_CAPABILITY_ID_MSI = 0x05; /* #define this. */ + uint8_t PCIE_CAPABILITY_ID_MSIX = 0x11; /* #define this. */ + uint8_t PCIE_CAPABILITY_ID_PM = 0x1; /* #define this. */ + if (cap_id == PCIE_CAPABILITY_ID_MSI) { + uint16_t message_control = capability[2] | ((uint16_t)capability[3] << 8); + sddf_dprintf("\tmessage control(msi): %x\n", message_control); + } else if (cap_id == PCIE_CAPABILITY_ID_MSIX) { + uint16_t message_control = capability[2] | ((uint16_t)capability[3] << 8); + sddf_dprintf("\tmessage control(msix): %x\n", message_control); + } else if (cap_id == PCIE_CAPABILITY_ID_PM) { + // https://lekensteyn.nl/files/docs/PCI_Power_Management_12.pdf¸ §3.2 + // not this... + uint16_t control = capability[4] | ((uint16_t)capability[5] << 8); + sddf_dprintf("\tpower management ctrl: %x\n", control); + } else if (cap_id == 0x10 /* PCI Express */) { + sddf_dprintf("\troot control(pcie): %x\n", *(uint16_t *)&(capability[0x1C])); + sddf_dprintf("\troot status(pcie): %x\n", *(uint32_t *)&(capability[0x20])); + } else { + sddf_dprintf("\tunknown\n"); + } + + current_ptr = (capability[1] & ~0b11); + } +} + void init() { sddf_dprintf("pcie driver starting!\n"); +#if 0 for (uint8_t bus = 0; bus <= 255; bus++) { for (uint8_t device = 0; device < 32; device++) { for (uint8_t function = 0; function < 8; function++) { @@ -166,12 +237,29 @@ void init() out: sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); +#endif + device_ack_irq(0, 0, 0, false); + device_ack_irq(1, 0, 0, false); nvme_init(); } void notified(microkit_channel ch) { + static int i = 0; sddf_dprintf("notified on ch: %u\n", ch); - microkit_irq_ack(ch); + + // if (i == 0) { + // device_print(1, 0, 0); + // i++; + // } + + if (i < 10) { + microkit_irq_ack(ch); + i++; + } else { + device_ack_irq(0, 0, 0, true); + device_ack_irq(1, 0, 0, true); + sddf_dprintf("stopped ACKing seL4\n"); + } } From 28edb4731d9c5dd0f0755ea4792690ebaf6d93be Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 14:13:52 +1100 Subject: [PATCH 39/60] try to setup qemu Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 36 ++- .../pcie/board/qemu_virt_riscv64/pcie.system | 44 ++++ examples/pcie/build.zig | 51 ++++- examples/pcie/qemu.dts | 208 ++++++++++++++++++ 4 files changed, 323 insertions(+), 16 deletions(-) create mode 100644 examples/pcie/board/qemu_virt_riscv64/pcie.system create mode 100644 examples/pcie/qemu.dts diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 56a7af5e4..9838e4c3b 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -36,6 +36,12 @@ uintptr_t get_bdf_offset(uint8_t bus, uint8_t device, uint8_t function) return offset; } +static bool found_nvme = false; +static uint8_t nvme_bus; +static uint8_t nvme_device; +static uint8_t nvme_function; +uintptr_t nvme_controller_paddr; + void device_print(uint8_t bus, uint8_t device, uint8_t function) { uintptr_t offset = get_bdf_offset(bus, device, function); @@ -69,6 +75,21 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("status register: 0x%04x\n", header->status); sddf_dprintf("revision ID: 0x%02x\n", header->revision_id); sddf_dprintf("base-class code: 0x%02x | sub-class code: 0x%02x\n", header->base_class_code, header->subclass_code); + if (header->base_class_code == 0x1 && header->subclass_code == 0x8) { + sddf_dprintf("FOUND NVME!!!\n"); + found_nvme = true; + nvme_bus = bus; + nvme_device = device; + nvme_function = function; + + // TODO: hacky + volatile pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; + header->command &= ~BIT(1); + type0_header->base_address_registers[0] = nvme_controller_paddr; + type0_header->base_address_registers[1] = 0x0; + header->command |= BIT(1); + + } sddf_dprintf("header type: 0x%02x\n", header->header_type); sddf_dprintf("\thas multi-functions: %s\n", @@ -177,7 +198,9 @@ void device_ack_irq(uint8_t bus, uint8_t device, uint8_t function, bool to_mask) // interrupt pin: 0 = no, 1 => INTA#, 2 => INTB#, 3 => INTC#, 4 => INTD# sddf_printf("\t(PIN-based) interrupt line: %02x, interrupt pin: %02x\n", type0_header->interrupt_line, type0_header->interrupt_pin); - assert(header->status & BIT(4)); /* assert capabilities list status exists */ + if (!(header->status & BIT(4))) { + sddf_dprintf("no capabilities??\n"); + }; /* assert capabilities list status exists */ sddf_dprintf("start capabilities PTR: %u\n", type0_header->capabilities_pointer); /* ach capability @@ -221,7 +244,7 @@ void init() { sddf_dprintf("pcie driver starting!\n"); -#if 0 +#if 1 for (uint8_t bus = 0; bus <= 255; bus++) { for (uint8_t device = 0; device < 32; device++) { for (uint8_t function = 0; function < 8; function++) { @@ -230,6 +253,8 @@ void init() goto out; } + /* TODO: This also configures BARs for the NVMe devices... */ + /* That is not printing. */ device_print(bus, device, function); } } @@ -239,8 +264,8 @@ void init() sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); #endif - device_ack_irq(0, 0, 0, false); - device_ack_irq(1, 0, 0, false); + assert(found_nvme); + device_ack_irq(nvme_bus, nvme_device, nvme_function, false); nvme_init(); } @@ -258,8 +283,7 @@ void notified(microkit_channel ch) microkit_irq_ack(ch); i++; } else { - device_ack_irq(0, 0, 0, true); - device_ack_irq(1, 0, 0, true); + device_ack_irq(nvme_bus, nvme_device, nvme_function, true); sddf_dprintf("stopped ACKing seL4\n"); } } diff --git a/examples/pcie/board/qemu_virt_riscv64/pcie.system b/examples/pcie/board/qemu_virt_riscv64/pcie.system new file mode 100644 index 000000000..617181274 --- /dev/null +++ b/examples/pcie/board/qemu_virt_riscv64/pcie.system @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index 5d4be4507..b911def32 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -6,6 +6,7 @@ const std = @import("std"); const MicrokitBoard = enum { star64, + qemu_virt_riscv64, }; const Target = struct { @@ -23,6 +24,15 @@ const targets = [_]Target{ .abi = .none, }, }, + .{ + .board = MicrokitBoard.qemu_virt_riscv64, + .zig_target = std.Target.Query{ + .cpu_arch = .riscv64, + .cpu_model = .{ .explicit = &std.Target.riscv.cpu.baseline_rv64 }, + .os_tag = .freestanding, + .abi = .none, + }, + }, }; fn findTarget(board: MicrokitBoard) std.Target.Query { @@ -78,9 +88,10 @@ pub fn build(b: *std.Build) void { const driver_class = switch (microkit_board_option.?) { .star64 => "starfive", + .qemu_virt_riscv64 => "starfive", // hack }; - const driver = sddf_dep.artifact(b.fmt("driver_pcie_{s}.elf", .{ driver_class })); + const driver = sddf_dep.artifact(b.fmt("driver_pcie_{s}.elf", .{driver_class})); // This is required because the SDF file is expecting a different name to the artifact we // are dealing with. const driver_install = b.addInstallArtifact(driver, .{ .dest_sub_path = "pcie_driver.elf" }); @@ -105,18 +116,38 @@ pub fn build(b: *std.Build) void { const system_description_path = b.fmt("board/{s}/pcie.system", .{microkit_board}); const final_image_dest = b.getInstallPath(.bin, "./loader.img"); - const microkit_tool_cmd = b.addSystemCommand(&[_][]const u8{ - microkit_tool, - system_description_path, - "--search-path", b.getInstallPath(.bin, ""), - "--board", microkit_board, - "--config", microkit_config, - "-o", final_image_dest, - "-r", b.getInstallPath(.prefix, "./report.txt") - }); + const microkit_tool_cmd = b.addSystemCommand(&[_][]const u8{ microkit_tool, system_description_path, "--search-path", b.getInstallPath(.bin, ""), "--board", microkit_board, "--config", microkit_config, "-o", final_image_dest, "-r", b.getInstallPath(.prefix, "./report.txt") }); microkit_tool_cmd.step.dependOn(b.getInstallStep()); microkit_tool_cmd.step.dependOn(&driver_install.step); const microkit_step = b.step("microkit", "Compile and build the final bootable image"); microkit_step.dependOn(µkit_tool_cmd.step); b.default_step = microkit_step; + + if (std.mem.eql(u8, microkit_board, "qemu_virt_riscv64")) { + const create_disk_cmd = b.addSystemCommand(&[_][]const u8{ + "bash", "../blk/mkvirtdisk", + }); + const disk = create_disk_cmd.addOutputFileArg("disk"); + create_disk_cmd.addArgs(&[_][]const u8{ + "1", "512", b.fmt("{}", .{ 1024 * 1024 * 16 }), + }); + const disk_install = b.addInstallFile(disk, "disk"); + disk_install.step.dependOn(&create_disk_cmd.step); + + const qemu_cmd = b.addSystemCommand(&[_][]const u8{ + "qemu-system-riscv64", + "-machine", "virt", + "-serial", "mon:stdio", + "-kernel", final_image_dest, + "-m", "size=2G", + "-nographic", + "-drive", b.fmt("file={s},if=none,format=raw,id=hd", .{ b.getInstallPath(.prefix, "disk") }), + "-device", "nvme,serial=deadbeef,drive=hd", + // "--trace", "events=/tmp/events", + }); + qemu_cmd.step.dependOn(b.default_step); + qemu_cmd.step.dependOn(&disk_install.step); + const simulate_step = b.step("qemu", "Simulate the image using QEMU"); + simulate_step.dependOn(&qemu_cmd.step); + } } diff --git a/examples/pcie/qemu.dts b/examples/pcie/qemu.dts new file mode 100644 index 000000000..1942c1a3d --- /dev/null +++ b/examples/pcie/qemu.dts @@ -0,0 +1,208 @@ +/dts-v1/; + +/ { + #address-cells = <0x02>; + #size-cells = <0x02>; + compatible = "riscv-virtio"; + model = "riscv-virtio,qemu"; + + poweroff { + value = <0x5555>; + offset = <0x00>; + regmap = <0x04>; + compatible = "syscon-poweroff"; + }; + + reboot { + value = <0x7777>; + offset = <0x00>; + regmap = <0x04>; + compatible = "syscon-reboot"; + }; + + platform-bus@4000000 { + interrupt-parent = <0x03>; + ranges = <0x00 0x00 0x4000000 0x2000000>; + #address-cells = <0x01>; + #size-cells = <0x01>; + compatible = "qemu,platform\0simple-bus"; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x00 0x80000000 0x00 0x80000000>; + }; + + cpus { + #address-cells = <0x01>; + #size-cells = <0x00>; + timebase-frequency = <0x989680>; + + cpu@0 { + phandle = <0x01>; + device_type = "cpu"; + reg = <0x00>; + status = "okay"; + compatible = "riscv"; + riscv,cboz-block-size = <0x40>; + riscv,cbom-block-size = <0x40>; + riscv,isa = "rv64imafdch_zicbom_zicboz_zicntr_zicsr_zifencei_zihintntl_zihintpause_zihpm_zawrs_zfa_zca_zcd_zba_zbb_zbc_zbs_sstc_svadu"; + mmu-type = "riscv,sv57"; + + interrupt-controller { + #interrupt-cells = <0x01>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + phandle = <0x02>; + }; + }; + + cpu-map { + + cluster0 { + + core0 { + cpu = <0x01>; + }; + }; + }; + }; + + pmu { + riscv,event-to-mhpmcounters = <0x01 0x01 0x7fff9 0x02 0x02 0x7fffc 0x10019 0x10019 0x7fff8 0x1001b 0x1001b 0x7fff8 0x10021 0x10021 0x7fff8>; + compatible = "riscv,pmu"; + }; + + fw-cfg@10100000 { + dma-coherent; + reg = <0x00 0x10100000 0x00 0x18>; + compatible = "qemu,fw-cfg-mmio"; + }; + + flash@20000000 { + bank-width = <0x04>; + reg = <0x00 0x20000000 0x00 0x2000000 0x00 0x22000000 0x00 0x2000000>; + compatible = "cfi-flash"; + }; + + chosen { + stdout-path = "/soc/serial@10000000"; + rng-seed = <0xeacbf6e6 0x149b4d83 0xa4525e73 0x367c75e 0xccb35a12 0x4172ecef 0x1b52148f 0x887a0864>; + }; + + soc { + #address-cells = <0x02>; + #size-cells = <0x02>; + compatible = "simple-bus"; + ranges; + + rtc@101000 { + interrupts = <0x0b>; + interrupt-parent = <0x03>; + reg = <0x00 0x101000 0x00 0x1000>; + compatible = "google,goldfish-rtc"; + }; + + serial@10000000 { + interrupts = <0x0a>; + interrupt-parent = <0x03>; + clock-frequency = "\08@"; + reg = <0x00 0x10000000 0x00 0x100>; + compatible = "ns16550a"; + }; + + test@100000 { + phandle = <0x04>; + reg = <0x00 0x100000 0x00 0x1000>; + compatible = "sifive,test1\0sifive,test0\0syscon"; + }; + + pci@30000000 { + interrupt-map-mask = <0x1800 0x00 0x00 0x07>; + interrupt-map = <0x00 0x00 0x00 0x01 0x03 0x20 0x00 0x00 0x00 0x02 0x03 0x21 0x00 0x00 0x00 0x03 0x03 0x22 0x00 0x00 0x00 0x04 0x03 0x23 0x800 0x00 0x00 0x01 0x03 0x21 0x800 0x00 0x00 0x02 0x03 0x22 0x800 0x00 0x00 0x03 0x03 0x23 0x800 0x00 0x00 0x04 0x03 0x20 0x1000 0x00 0x00 0x01 0x03 0x22 0x1000 0x00 0x00 0x02 0x03 0x23 0x1000 0x00 0x00 0x03 0x03 0x20 0x1000 0x00 0x00 0x04 0x03 0x21 0x1800 0x00 0x00 0x01 0x03 0x23 0x1800 0x00 0x00 0x02 0x03 0x20 0x1800 0x00 0x00 0x03 0x03 0x21 0x1800 0x00 0x00 0x04 0x03 0x22>; + ranges = <0x1000000 0x00 0x00 0x00 0x3000000 0x00 0x10000 0x2000000 0x00 0x40000000 0x00 0x40000000 0x00 0x40000000 0x3000000 0x04 0x00 0x04 0x00 0x04 0x00>; + reg = <0x00 0x30000000 0x00 0x10000000>; + dma-coherent; + bus-range = <0x00 0xff>; + linux,pci-domain = <0x00>; + device_type = "pci"; + compatible = "pci-host-ecam-generic"; + #size-cells = <0x02>; + #interrupt-cells = <0x01>; + #address-cells = <0x03>; + }; + + virtio_mmio@10008000 { + interrupts = <0x08>; + interrupt-parent = <0x03>; + reg = <0x00 0x10008000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10007000 { + interrupts = <0x07>; + interrupt-parent = <0x03>; + reg = <0x00 0x10007000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10006000 { + interrupts = <0x06>; + interrupt-parent = <0x03>; + reg = <0x00 0x10006000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10005000 { + interrupts = <0x05>; + interrupt-parent = <0x03>; + reg = <0x00 0x10005000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10004000 { + interrupts = <0x04>; + interrupt-parent = <0x03>; + reg = <0x00 0x10004000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10003000 { + interrupts = <0x03>; + interrupt-parent = <0x03>; + reg = <0x00 0x10003000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10002000 { + interrupts = <0x02>; + interrupt-parent = <0x03>; + reg = <0x00 0x10002000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@10001000 { + interrupts = <0x01>; + interrupt-parent = <0x03>; + reg = <0x00 0x10001000 0x00 0x1000>; + compatible = "virtio,mmio"; + }; + + plic@c000000 { + phandle = <0x03>; + riscv,ndev = <0x5f>; + reg = <0x00 0xc000000 0x00 0x600000>; + interrupts-extended = <0x02 0x0b 0x02 0x09>; + interrupt-controller; + compatible = "sifive,plic-1.0.0\0riscv,plic0"; + #address-cells = <0x00>; + #interrupt-cells = <0x01>; + }; + + clint@2000000 { + interrupts-extended = <0x02 0x03 0x02 0x07>; + reg = <0x00 0x2000000 0x00 0x10000>; + compatible = "sifive,clint0\0riscv,clint0"; + }; + }; +}; From 2803b7156068f7d80df1ae7af09ef3286d2ec889 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 14:39:48 +1100 Subject: [PATCH 40/60] qemu reading 0xfffff from nvme Signed-off-by: julia --- drivers/nvme/nvme.c | 7 ++++++- drivers/pcie/starfive/pcie.c | 4 +++- examples/pcie/build.zig | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index ddf8ff241..16b0fc5df 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -47,7 +47,12 @@ void nvme_controller_init() nvme_controller->cc &= ~NVME_CC_EN; // 1. Wait for CSTS.RDY to become '0' (i.e. not ready) - while (nvme_controller->csts & NVME_CSTS_RDY); + int i = 100; + while (nvme_controller->csts & NVME_CSTS_RDY && i != 0) i--; + if (i == 0) { + sddf_dprintf("time out\n"); + return; + } // 2. Configure Admin Queue(s); nvme_queues_init(&admin_queue, /* y */ 0, nvme_controller, nvme_asq_region, NVME_ASQ_CAPACITY, nvme_acq_region, diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 9838e4c3b..b8590fecb 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -69,6 +69,8 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) } sddf_dprintf("\nB.D:F: %02x:%02x.%01x\n", bus, device, function); + // enable bus mastering... and memory accesses + header->command |= BIT(2) | BIT(1); sddf_dprintf("vendor ID: 0x%04x\n", header->vendor_id); sddf_dprintf("device ID: 0x%04x\n", header->device_id); sddf_dprintf("command register: 0x%04x\n", header->command); @@ -88,7 +90,6 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) type0_header->base_address_registers[0] = nvme_controller_paddr; type0_header->base_address_registers[1] = 0x0; header->command |= BIT(1); - } sddf_dprintf("header type: 0x%02x\n", header->header_type); @@ -96,6 +97,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) header->header_type & PCIE_HEADER_TYPE_HAS_MULTI_FUNCTIONS ? "yes" : "no"); sddf_dprintf("\tlayout variant: 0x%02lx\n", header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK); + if ((header->header_type & PCIE_HEADER_TYPE_LAYOUT_MASK) == PCIE_HEADER_TYPE_GENERAL) { volatile pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; for (int i = 0; i < 6; i++) { diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index b911def32..4a2df283d 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -141,9 +141,12 @@ pub fn build(b: *std.Build) void { "-kernel", final_image_dest, "-m", "size=2G", "-nographic", + "-d", "guest_errors", "-drive", b.fmt("file={s},if=none,format=raw,id=hd", .{ b.getInstallPath(.prefix, "disk") }), "-device", "nvme,serial=deadbeef,drive=hd", - // "--trace", "events=/tmp/events", + // "--trace", "pci*", + // "--trace", "pci_nvme*", + // "--trace", "nvme*", }); qemu_cmd.step.dependOn(b.default_step); qemu_cmd.step.dependOn(&disk_install.step); From 41ec88958d3cac9de599ea42070e2f633b6284cb Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 14:55:25 +1100 Subject: [PATCH 41/60] get back to original state on star64 (still not working tho) Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 1 + examples/pcie/board/star64/pcie.system | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index b8590fecb..012870778 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -85,6 +85,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) nvme_function = function; // TODO: hacky + assert(nvme_controller_paddr != 0); volatile pcie_header_type0_t *type0_header = (pcie_header_type0_t *)config_base; header->command &= ~BIT(1); type0_header->base_address_registers[0] = nvme_controller_paddr; diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index 8f409d136..df32bd803 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -25,6 +25,7 @@ + From b00a8e530e4247bf9f22131383b6ac1a4ffa4dd5 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 15:16:05 +1100 Subject: [PATCH 42/60] nvme regs on qemu have to be above 0x4000_0000 Signed-off-by: julia --- examples/pcie/board/qemu_virt_riscv64/pcie.system | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/pcie/board/qemu_virt_riscv64/pcie.system b/examples/pcie/board/qemu_virt_riscv64/pcie.system index 617181274..440289f5c 100644 --- a/examples/pcie/board/qemu_virt_riscv64/pcie.system +++ b/examples/pcie/board/qemu_virt_riscv64/pcie.system @@ -5,11 +5,10 @@ SPDX-License-Identifier: BSD-2-Clause --> - + - - + + From 15de7a5ac2f63ec9922e48dd62b79c34901f59c5 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 15:47:58 +1100 Subject: [PATCH 43/60] remove unaligned reads & writes to doorbells these are undefined by PCIe/NVMe specs Signed-off-by: julia --- drivers/nvme/nvme.c | 7 ++++--- drivers/nvme/nvme_queue.h | 19 +++++++------------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 16b0fc5df..33a92bba0 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -40,8 +40,9 @@ _Static_assert(NVME_IO_CQ_CAPACITY <= 0x10000, "capacity of IO CQ must be <=6553 void nvme_controller_init() { LOG_NVME("CAP: %016lx\n", nvme_controller->cap); - LOG_NVME("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, - nvme_controller->vs.ter); + // TODO: alignment 32-bit. + // LOG_NVME("VS: major: %u, minor: %u, tertiary: %u\n", nvme_controller->vs.mjr, nvme_controller->vs.mnr, + // nvme_controller->vs.ter); LOG_NVME("CC: %08x\n", nvme_controller->cc); nvme_controller->cc &= ~NVME_CC_EN; @@ -198,7 +199,7 @@ void nvme_init() assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field // for (int i = 0; i < 32; i++) { - // sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); + // sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); // } for (int i = 0; i < 4096; i++) { diff --git a/drivers/nvme/nvme_queue.h b/drivers/nvme/nvme_queue.h index dbd10570c..911c5cce5 100644 --- a/drivers/nvme/nvme_queue.h +++ b/drivers/nvme/nvme_queue.h @@ -28,22 +28,23 @@ static inline void nvme_queues_init(nvme_queue_info_t *queue, uint16_t y, volati { uint8_t DSTRD = (nvme_controller->cap & NVME_CAP_DSTRD_MASK) >> NVME_CAP_DSTRD_SHIFT; - /* [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL */ + /* [NVMEe-Transport-PCIe-1.1] 3.1.2.1 SQyTDBL & 3.1.2.2 CQyHDBL + Note: "The host should not read the doorbell registers" + */ volatile uint32_t *submission_doorbell = (void *)nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y, DSTRD); - /* [NVMEe-Transport-PCIe-1.1] 3.1.2.2 CQyHDBL */ volatile uint32_t *completion_doorbell = (void *)nvme_controller + NVME_PCIE_DOORBELL_OFFSET(2 * y + 1, DSTRD); *queue = (nvme_queue_info_t){ .submission = { .queue = submission_queue, .capacity = submission_capacity, - .tail = *submission_doorbell & NVME_PCIE_SQT_MASK, + .tail = 0, .doorbell = submission_doorbell, }, .completion = { .queue = completion_queue, .capacity = completion_capacity, - .head = *completion_doorbell & NVME_PCIE_CQH_MASK, + .head = 0, .doorbell = completion_doorbell, .phase = 0, }, @@ -57,10 +58,7 @@ static inline void nvme_queue_submit(nvme_queue_info_t *queue, nvme_submission_q // todo: overflow. todo: full queue. queue->submission.tail++; - uint32_t val = *queue->submission.doorbell; - val &= ~NVME_PCIE_SQT_MASK; - val |= queue->submission.tail & NVME_PCIE_SQT_MASK; - *queue->submission.doorbell = val; + *queue->submission.doorbell = queue->submission.tail; } static inline int nvme_queue_consume(nvme_queue_info_t *queue, nvme_completion_queue_entry_t *entry) @@ -80,10 +78,7 @@ static inline int nvme_queue_consume(nvme_queue_info_t *queue, nvme_completion_q // phase ^= 1; /* flip phase */ // } - uint32_t val = *queue->completion.doorbell; - val &= ~NVME_PCIE_CQH_MASK; - val |= queue->completion.head & NVME_PCIE_CQH_MASK; - *queue->completion.doorbell = val; + *queue->completion.doorbell = queue->completion.head; return 0; } From b950c18cf51b8fa72201b0d6e754a5ae503b2849 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 17:04:59 +1100 Subject: [PATCH 44/60] controller caps broken in qemu Signed-off-by: julia --- drivers/nvme/nvme.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 33a92bba0..d1e2d01bd 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -67,14 +67,19 @@ void nvme_controller_init() | ((NVME_ACQ_CAPACITY - 1) << NVME_AQA_ACQS_SHIFT); // 3. Initialise Command Support Sets. - nvme_controller->cc &= ~(NVME_CC_CSS_MASK); - if (nvme_controller->cap & NVME_CAP_NOIOCSS) { - nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; - } else if (nvme_controller->cap & NVME_CAP_IOCSS) { - nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; - } else if (nvme_controller->cap & NVME_CAP_NCSS) { - nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; - } + /* + https://forum.osdev.org/viewtopic.php?p=343454#p343454 + For whatever reason this is broken under QEMU. + + nvme_controller->cc &= ~(NVME_CC_CSS_MASK); + if (nvme_controller->cap & NVME_CAP_NOIOCSS) { + nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_IOCSS) { + nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_NCSS) { + nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; + } + */ // 4a. Arbitration Mechanism (TODO) // 4b. Memory Page Size From 673023f4c0fa03f123fae16fee78f82563c256ec Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 17:44:06 +1100 Subject: [PATCH 45/60] qemu IRQ! Signed-off-by: julia --- examples/pcie/board/qemu_virt_riscv64/pcie.system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pcie/board/qemu_virt_riscv64/pcie.system b/examples/pcie/board/qemu_virt_riscv64/pcie.system index 440289f5c..5f8200059 100644 --- a/examples/pcie/board/qemu_virt_riscv64/pcie.system +++ b/examples/pcie/board/qemu_virt_riscv64/pcie.system @@ -20,7 +20,7 @@ - + From 7b16ddfcf768f224d6f10a7dd83a29d45471359c Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 15 Oct 2024 18:21:40 +1100 Subject: [PATCH 46/60] qemu interrupt-y style. I have no idea how this works (it only interrupt sonce???? even though it should do forever???) https://www.mail-archive.com/qemu-devel@nongnu.org/msg931360.html might be this bug Signed-off-by: julia --- drivers/nvme/nvme.c | 73 +++++++++++++++++------------------- drivers/pcie/starfive/pcie.c | 22 ++++++----- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index d1e2d01bd..30e8efccb 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -116,10 +116,6 @@ void nvme_controller_init() assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - nvme_controller->intms = 0x00000001; - /* At this point: we start getting interrupts */ - return; - // 8. The host determines any I/O Command Set specific configuration information // TODO: Why??? @@ -185,43 +181,42 @@ void nvme_init() // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 nvme_controller_init(); - return; - - for (int z = 0; z < 9; z++) { - /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ - nvme_completion_queue_entry_t entry; - uint16_t number_blocks = 1; - entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, - .nsid = 0x1, // TOOD: Why is NSID 1 now ???? - .cdw10 = /* SLBA[31:00] */ 0x0, - .cdw11 = /* SLBA[63:32] */ 0x0, - .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), - .prp2 = 0x0, - .prp1 = data_region_paddr, - }); - - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - - // for (int i = 0; i < 32; i++) { - // sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); - // } - - for (int i = 0; i < 4096; i++) { - data_region[i] = data_region[i] ^ 0xbb; - } +} + +void nvme_continue(int z) +{ + /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ + nvme_completion_queue_entry_t entry; + uint16_t number_blocks = 1; + entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), + .prp2 = 0x0, + .prp1 = data_region_paddr, + }); - entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1101 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, - .nsid = 0x1, // TOOD: Why is NSID 1 now ???? - .cdw10 = /* SLBA[31:00] */ 0x0, - .cdw11 = /* SLBA[63:32] */ 0x0, - .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), - .prp2 = 0x0, - .prp1 = data_region_paddr, - }); + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + // for (int i = 0; i < 32; i++) { + // sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); + // } + for (int i = 0; i < 4096; i++) { + data_region[i] = data_region[i] ^ 0xbb; } + + entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1101 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), + .prp2 = 0x0, + .prp1 = data_region_paddr, + }); + + assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field } diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 012870778..d4873fa9a 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -77,7 +77,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) sddf_dprintf("status register: 0x%04x\n", header->status); sddf_dprintf("revision ID: 0x%02x\n", header->revision_id); sddf_dprintf("base-class code: 0x%02x | sub-class code: 0x%02x\n", header->base_class_code, header->subclass_code); - if (header->base_class_code == 0x1 && header->subclass_code == 0x8) { + if (header->base_class_code == 0x1 && header->subclass_code == 0x8 && !found_nvme) { sddf_dprintf("FOUND NVME!!!\n"); found_nvme = true; nvme_bus = bus; @@ -272,21 +272,25 @@ void init() nvme_init(); } +void nvme_continue(int z); void notified(microkit_channel ch) { static int i = 0; sddf_dprintf("notified on ch: %u\n", ch); - // if (i == 0) { - // device_print(1, 0, 0); - // i++; - // } - if (i < 10) { + sddf_dprintf("acking (%u)\n", i); + // // device_print(nvme_bus, nvme_device, nvme_function); + // device_ack_irq(nvme_bus, nvme_device, nvme_function, false); + microkit_irq_ack(ch); + + nvme_continue(i); + } else if (i < 20) { + sddf_dprintf("\nhopefully stopping\n"); microkit_irq_ack(ch); - i++; } else { - device_ack_irq(nvme_bus, nvme_device, nvme_function, true); - sddf_dprintf("stopped ACKing seL4\n"); + sddf_dprintf("stopping\n"); } + + i++; } From 898d0ae4001dd9d72b3bb33316f7ce78c967bd94 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Oct 2024 13:25:03 +1100 Subject: [PATCH 47/60] ifdef the QEMU-specific behaviour Signed-off-by: julia --- drivers/nvme/nvme.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 30e8efccb..085dae8df 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -67,19 +67,23 @@ void nvme_controller_init() | ((NVME_ACQ_CAPACITY - 1) << NVME_AQA_ACQS_SHIFT); // 3. Initialise Command Support Sets. + nvme_controller->cc &= ~(NVME_CC_CSS_MASK); + if (nvme_controller->cap & NVME_CAP_NOIOCSS) { + nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_IOCSS) { + nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; + } else if (nvme_controller->cap & NVME_CAP_NCSS) { + nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; + } + +#if defined(CONFIG_PLAT_QEMU_RISCV_VIRT) /* - https://forum.osdev.org/viewtopic.php?p=343454#p343454 - For whatever reason this is broken under QEMU. - - nvme_controller->cc &= ~(NVME_CC_CSS_MASK); - if (nvme_controller->cap & NVME_CAP_NOIOCSS) { - nvme_controller->cc |= 0b111 << NVME_CC_CSS_SHIFT; - } else if (nvme_controller->cap & NVME_CAP_IOCSS) { - nvme_controller->cc |= 0b110 << NVME_CC_CSS_SHIFT; - } else if (nvme_controller->cap & NVME_CAP_NCSS) { - nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; - } + QEMU deviates from the NVMe specification: + https://gitlab.com/qemu-project/qemu/-/issues/1691 */ + nvme_controller->cc &= ~(NVME_CC_CSS_MASK); + nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; +#endif // 4a. Arbitration Mechanism (TODO) // 4b. Memory Page Size @@ -101,7 +105,9 @@ void nvme_controller_init() /* https://github.com/redox-os/drivers/blob/master/storage/nvmed/src/nvme/mod.rs#L292*/ nvme_controller->intms = 0xFFFFFFFF; + /* TODO: Somethigng about this. See PCIe transport spec. */ nvme_controller->intmc = 0x00000001; + // nvme_controller->intms = 0x00000001; // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 // TODO: What do we actually need this for???? From 62f64f28df0d36a0d3c57862529a7cfd6fef5657 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Oct 2024 13:37:22 +1100 Subject: [PATCH 48/60] check LE cpu Signed-off-by: julia --- drivers/nvme/nvme.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 085dae8df..56abe6028 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -8,6 +8,10 @@ #include "nvme_debug.h" #endif +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error "code assumes little endian CPU as NVMe/PCIe is little endian" +#endif + volatile nvme_controller_t *nvme_controller; nvme_submission_queue_entry_t *nvme_asq_region; nvme_completion_queue_entry_t *nvme_acq_region; From 4ca9808f8d3b1eefc289462f9dcdab72c915a235 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Oct 2024 15:29:33 +1100 Subject: [PATCH 49/60] interrupts work in QEMU, at least Signed-off-by: julia --- drivers/nvme/nvme.c | 122 +++++++++++++++++++++++------------ drivers/pcie/starfive/pcie.c | 2 +- 2 files changed, 82 insertions(+), 42 deletions(-) diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index 56abe6028..f3a423049 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -40,6 +40,34 @@ _Static_assert(NVME_ACQ_CAPACITY <= 0x1000, "capacity of ACQ must be <=4096 (ent _Static_assert(NVME_IO_SQ_CAPACITY <= 0x10000, "capacity of IO SQ must be <=65536 (entries)"); _Static_assert(NVME_IO_CQ_CAPACITY <= 0x10000, "capacity of IO CQ must be <=65536 (entries)"); +void nvme_irq_mask(void) +{ + /* [NVMe-Transport-PCIe-1.1] 3.5.1.1 Differences between Pin Based and MSI Interrupts + > Pin-based and single MSI only use one interrupt vector. + > Multiple MSI may use up to 32 interrupt vectors. + + [NVMe-2.1] 3.1.4.10 Admin Completion Queue Base Address + > This queue is always associated with interrupt vector 0. + */ + + /* For now -- we mask out every interrupt vector) */ + nvme_controller->intms = 0xffffffff; +} + +void nvme_irq_unmask(void) +{ + /* [NVMe-Transport-PCIe-1.1] 3.5.1.1 Differences between Pin Based and MSI Interrupts + > Pin-based and single MSI only use one interrupt vector. + > Multiple MSI may use up to 32 interrupt vectors. + + [NVMe-2.1] 3.1.4.10 Admin Completion Queue Base Address + > This queue is always associated with interrupt vector 0. + */ + + /* For now -- we mask in only vector 0, as it's the only one */ + nvme_controller->intmc = 0xffffffff; +} + /* [NVMe-2.1] 3.5.1 Memory-based Controller Initialization (PCIe) */ void nvme_controller_init() { @@ -62,6 +90,7 @@ void nvme_controller_init() // 2. Configure Admin Queue(s); nvme_queues_init(&admin_queue, /* y */ 0, nvme_controller, nvme_asq_region, NVME_ASQ_CAPACITY, nvme_acq_region, NVME_ACQ_CAPACITY); + nvme_irq_mask(); assert(nvme_asq_region_paddr != 0x0); assert(nvme_acq_region_paddr != 0x0); nvme_controller->asq = nvme_asq_region_paddr; @@ -107,12 +136,6 @@ void nvme_controller_init() while (!(nvme_controller->csts & NVME_CSTS_RDY)); LOG_NVME("\tdone\n"); - /* https://github.com/redox-os/drivers/blob/master/storage/nvmed/src/nvme/mod.rs#L292*/ - nvme_controller->intms = 0xFFFFFFFF; - /* TODO: Somethigng about this. See PCIe transport spec. */ - nvme_controller->intmc = 0x00000001; - // nvme_controller->intms = 0x00000001; - // 7. Send the Identify Controller command (Identify with CNS = 01h); §5.1.13 // TODO: What do we actually need this for???? // sudo nvme admin-passthru /dev/nvme0 --opcode=0x06 --cdw10=0x0001 --data-len=4096 -r -s @@ -154,7 +177,7 @@ void nvme_controller_init() entry = nvme_queue_submit_and_consume_poll(&admin_queue, &(nvme_submission_queue_entry_t){ .cdw0 = /* CID */ (0b1010 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x5, .cdw10 = /* QSIZE */ ((NVME_IO_CQ_CAPACITY - 1U) << 16) | /* QID */ io_queue_id, - .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 0 << 1 | /* PC */ 0x1, + .cdw11 = /* IV */ (0x0 << 16) | /* IEN */ 1 << 1 | /* PC */ 0x1, .prp2 = 0, .prp1 = nvme_io_cq_region_paddr, }); @@ -180,53 +203,70 @@ void nvme_controller_init() // should submit an appropriate number of Asynchronous Event Request commands. This step may // be done at any point after the controller signals that the controller is ready (i.e., CSTS.RDY is set to ‘1’). // TODO: ??? + + nvme_irq_unmask(); } +void nvme_continue(int z); void nvme_init() { - LOG_NVME("Starting NVME config... (%s)\n", microkit_name); + LOG_NVME("Starting NVME initialisation... (%s)\n", microkit_name); // We should do a Function Level Reset as defined by [PCIe-2.0] spec §6.6.2 // https://github.com/bootreer/vroom/blob/d8bbe9db2b1cfdfc38eec31f3b48f5eb167879a9/src/nvme.rs#L220 nvme_controller_init(); + LOG_NVME("NVME initialised\n"); + + /* TODO: Don't send via this */ + nvme_continue(0); } +#define NUMBER_BLOCKS 1 void nvme_continue(int z) { - /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ - nvme_completion_queue_entry_t entry; - uint16_t number_blocks = 1; - entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, - .nsid = 0x1, // TOOD: Why is NSID 1 now ???? - .cdw10 = /* SLBA[31:00] */ 0x0, - .cdw11 = /* SLBA[63:32] */ 0x0, - .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), - .prp2 = 0x0, - .prp1 = data_region_paddr, - }); - - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field - - // for (int i = 0; i < 32; i++) { - // sddf_dprintf("Data [%02x]: %02x\n", i, data_region[i]); - // } - - for (int i = 0; i < 4096; i++) { - data_region[i] = data_region[i] ^ 0xbb; + if (z == 0) { + /* [NVMe-CommandSet-1.1] 3.3.4 Read command */ + nvme_queue_submit(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1011 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x2, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (NUMBER_BLOCKS - 1), + .prp2 = 0x0, + .prp1 = data_region_paddr, + }); + } else if (z == 1) { + nvme_completion_queue_entry_t cq_entry; + int ret = nvme_queue_consume(&io_queue, &cq_entry); + assert(ret == 0); + assert((cq_entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + for (int i = 0; i < 8; i++) { + LOG_NVME("Data [%02x]: %02x\n", i, data_region[i]); + } + + for (int i = 0; i < 4096; i++) { + data_region[i] = data_region[i] ^ 0xbb; + } + + /* [NVMe-CommandSet-1.1] ??????? write */ + nvme_queue_submit(&io_queue, &(nvme_submission_queue_entry_t){ + .cdw0 = /* CID */ (0b1101 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, + .nsid = 0x1, // TOOD: Why is NSID 1 now ???? + .cdw10 = /* SLBA[31:00] */ 0x0, + .cdw11 = /* SLBA[63:32] */ 0x0, + .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (NUMBER_BLOCKS - 1), + .prp2 = 0x0, + .prp1 = data_region_paddr, + }); + } else if (z == 2) { + nvme_completion_queue_entry_t cq_entry; + int ret = nvme_queue_consume(&io_queue, &cq_entry); + assert(ret == 0); + assert((cq_entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field + + LOG_NVME("Got response for write!\n"); } - - entry = nvme_queue_submit_and_consume_poll(&io_queue, &(nvme_submission_queue_entry_t){ - .cdw0 = /* CID */ (0b1101 << 16) | /* PSDT */ 0 | /* FUSE */ 0 | /* OPC */ 0x1, - .nsid = 0x1, // TOOD: Why is NSID 1 now ???? - .cdw10 = /* SLBA[31:00] */ 0x0, - .cdw11 = /* SLBA[63:32] */ 0x0, - .cdw12 = /* LR */ (0b1U << 31) | /* others */ 0 | /* NLB */ (number_blocks - 1), - .prp2 = 0x0, - .prp1 = data_region_paddr, - }); - - assert((entry.phase_tag_and_status & _MASK(1, 15)) == 0x0); // §4.2.3 Status Field } diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index d4873fa9a..460bf1b65 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -275,7 +275,7 @@ void init() void nvme_continue(int z); void notified(microkit_channel ch) { - static int i = 0; + static int i = 1; sddf_dprintf("notified on ch: %u\n", ch); if (i < 10) { From 503fbcd1a64a4649ec47b6e76eb820d6a6fcd683 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Oct 2024 16:17:11 +1100 Subject: [PATCH 50/60] yes QEMU on riscv is broken in multiple ways Signed-off-by: julia --- drivers/nvme/nvme.c | 12 +- .../pcie/board/qemu_virt_aarch64/pcie.system | 41 ++ examples/pcie/build.zig | 43 +- examples/pcie/qemu_arm.dts | 389 ++++++++++++++++++ 4 files changed, 482 insertions(+), 3 deletions(-) create mode 100644 examples/pcie/board/qemu_virt_aarch64/pcie.system create mode 100644 examples/pcie/qemu_arm.dts diff --git a/drivers/nvme/nvme.c b/drivers/nvme/nvme.c index f3a423049..c352b1e2a 100644 --- a/drivers/nvme/nvme.c +++ b/drivers/nvme/nvme.c @@ -109,7 +109,7 @@ void nvme_controller_init() nvme_controller->cc |= 0b000 << NVME_CC_CSS_SHIFT; } -#if defined(CONFIG_PLAT_QEMU_RISCV_VIRT) +#if defined(CONFIG_PLAT_QEMU_RISCV_VIRT) || defined(CONFIG_PLAT_QEMU_ARM_VIRT) /* QEMU deviates from the NVMe specification: https://gitlab.com/qemu-project/qemu/-/issues/1691 @@ -238,6 +238,14 @@ void nvme_continue(int z) .prp1 = data_region_paddr, }); } else if (z == 1) { + sddf_dprintf("doing nothing :P -- should get another IRQ \n"); + /* + So this works fine on QEMU AArch64... + but on QEMU RISCV level interrupts only get triggerred once + ... this caused issues in linux + ... https://www.mail-archive.com/qemu-devel@nongnu.org/msg931360.html + */ + } else if (z == 2) { nvme_completion_queue_entry_t cq_entry; int ret = nvme_queue_consume(&io_queue, &cq_entry); assert(ret == 0); @@ -261,7 +269,7 @@ void nvme_continue(int z) .prp2 = 0x0, .prp1 = data_region_paddr, }); - } else if (z == 2) { + } else if (z == 3) { nvme_completion_queue_entry_t cq_entry; int ret = nvme_queue_consume(&io_queue, &cq_entry); assert(ret == 0); diff --git a/examples/pcie/board/qemu_virt_aarch64/pcie.system b/examples/pcie/board/qemu_virt_aarch64/pcie.system new file mode 100644 index 000000000..754e7fd77 --- /dev/null +++ b/examples/pcie/board/qemu_virt_aarch64/pcie.system @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index 4a2df283d..8fcca94d7 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -6,6 +6,7 @@ const std = @import("std"); const MicrokitBoard = enum { star64, + qemu_virt_aarch64, qemu_virt_riscv64, }; @@ -24,6 +25,15 @@ const targets = [_]Target{ .abi = .none, }, }, + .{ + .board = MicrokitBoard.qemu_virt_aarch64, + .zig_target = std.Target.Query{ + .cpu_arch = .aarch64, + .cpu_model = .{ .explicit = &std.Target.arm.cpu.cortex_a53 }, + .os_tag = .freestanding, + .abi = .none, + }, + }, .{ .board = MicrokitBoard.qemu_virt_riscv64, .zig_target = std.Target.Query{ @@ -88,6 +98,7 @@ pub fn build(b: *std.Build) void { const driver_class = switch (microkit_board_option.?) { .star64 => "starfive", + .qemu_virt_aarch64 => "starfive", // hack .qemu_virt_riscv64 => "starfive", // hack }; @@ -123,7 +134,37 @@ pub fn build(b: *std.Build) void { microkit_step.dependOn(µkit_tool_cmd.step); b.default_step = microkit_step; - if (std.mem.eql(u8, microkit_board, "qemu_virt_riscv64")) { + if (std.mem.eql(u8, microkit_board, "qemu_virt_aarch64")) { + const create_disk_cmd = b.addSystemCommand(&[_][]const u8{ + "bash", "../blk/mkvirtdisk", + }); + const disk = create_disk_cmd.addOutputFileArg("disk"); + create_disk_cmd.addArgs(&[_][]const u8{ + "1", "512", b.fmt("{}", .{ 1024 * 1024 * 16 }), + }); + const disk_install = b.addInstallFile(disk, "disk"); + disk_install.step.dependOn(&create_disk_cmd.step); + + const qemu_cmd = b.addSystemCommand(&[_][]const u8{ + "qemu-system-aarch64", + "-machine", "virt,virtualization=on,highmem=off,secure=off", + "-cpu", "cortex-a53", + "-serial", "mon:stdio", + "-device", b.fmt("loader,file={s},addr=0x70000000,cpu-num=0", .{final_image_dest}), + "-m", "size=2G", + "-nographic", + "-d", "guest_errors", + "-drive", b.fmt("file={s},if=none,format=raw,id=hd", .{ b.getInstallPath(.prefix, "disk") }), + "-device", "nvme,serial=deadbeef,drive=hd", + // "--trace", "pci*", + // "--trace", "pci_nvme*", + // "--trace", "nvme*", + }); + qemu_cmd.step.dependOn(b.default_step); + qemu_cmd.step.dependOn(&disk_install.step); + const simulate_step = b.step("qemu", "Simulate the image using QEMU"); + simulate_step.dependOn(&qemu_cmd.step); + } else if (std.mem.eql(u8, microkit_board, "qemu_virt_riscv64")) { const create_disk_cmd = b.addSystemCommand(&[_][]const u8{ "bash", "../blk/mkvirtdisk", }); diff --git a/examples/pcie/qemu_arm.dts b/examples/pcie/qemu_arm.dts new file mode 100644 index 000000000..57e20a4b1 --- /dev/null +++ b/examples/pcie/qemu_arm.dts @@ -0,0 +1,389 @@ +/dts-v1/; + +/ { + interrupt-parent = <0x8002>; + model = "linux,dummy-virt"; + #size-cells = <0x02>; + #address-cells = <0x02>; + compatible = "linux,dummy-virt"; + + psci { + migrate = <0xc4000005>; + cpu_on = <0xc4000003>; + cpu_off = <0x84000002>; + cpu_suspend = <0xc4000001>; + method = "smc"; + compatible = "arm,psci-1.0\0arm,psci-0.2\0arm,psci"; + }; + + memory@40000000 { + reg = <0x00 0x40000000 0x00 0x80000000>; + device_type = "memory"; + }; + + platform-bus@c000000 { + interrupt-parent = <0x8002>; + ranges = <0x00 0x00 0xc000000 0x2000000>; + #address-cells = <0x01>; + #size-cells = <0x01>; + compatible = "qemu,platform\0simple-bus"; + }; + + fw-cfg@9020000 { + dma-coherent; + reg = <0x00 0x9020000 0x00 0x18>; + compatible = "qemu,fw-cfg-mmio"; + }; + + virtio_mmio@a000000 { + dma-coherent; + interrupts = <0x00 0x10 0x01>; + reg = <0x00 0xa000000 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000200 { + dma-coherent; + interrupts = <0x00 0x11 0x01>; + reg = <0x00 0xa000200 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000400 { + dma-coherent; + interrupts = <0x00 0x12 0x01>; + reg = <0x00 0xa000400 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000600 { + dma-coherent; + interrupts = <0x00 0x13 0x01>; + reg = <0x00 0xa000600 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000800 { + dma-coherent; + interrupts = <0x00 0x14 0x01>; + reg = <0x00 0xa000800 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000a00 { + dma-coherent; + interrupts = <0x00 0x15 0x01>; + reg = <0x00 0xa000a00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000c00 { + dma-coherent; + interrupts = <0x00 0x16 0x01>; + reg = <0x00 0xa000c00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a000e00 { + dma-coherent; + interrupts = <0x00 0x17 0x01>; + reg = <0x00 0xa000e00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001000 { + dma-coherent; + interrupts = <0x00 0x18 0x01>; + reg = <0x00 0xa001000 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001200 { + dma-coherent; + interrupts = <0x00 0x19 0x01>; + reg = <0x00 0xa001200 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001400 { + dma-coherent; + interrupts = <0x00 0x1a 0x01>; + reg = <0x00 0xa001400 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001600 { + dma-coherent; + interrupts = <0x00 0x1b 0x01>; + reg = <0x00 0xa001600 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001800 { + dma-coherent; + interrupts = <0x00 0x1c 0x01>; + reg = <0x00 0xa001800 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001a00 { + dma-coherent; + interrupts = <0x00 0x1d 0x01>; + reg = <0x00 0xa001a00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001c00 { + dma-coherent; + interrupts = <0x00 0x1e 0x01>; + reg = <0x00 0xa001c00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a001e00 { + dma-coherent; + interrupts = <0x00 0x1f 0x01>; + reg = <0x00 0xa001e00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002000 { + dma-coherent; + interrupts = <0x00 0x20 0x01>; + reg = <0x00 0xa002000 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002200 { + dma-coherent; + interrupts = <0x00 0x21 0x01>; + reg = <0x00 0xa002200 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002400 { + dma-coherent; + interrupts = <0x00 0x22 0x01>; + reg = <0x00 0xa002400 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002600 { + dma-coherent; + interrupts = <0x00 0x23 0x01>; + reg = <0x00 0xa002600 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002800 { + dma-coherent; + interrupts = <0x00 0x24 0x01>; + reg = <0x00 0xa002800 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002a00 { + dma-coherent; + interrupts = <0x00 0x25 0x01>; + reg = <0x00 0xa002a00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002c00 { + dma-coherent; + interrupts = <0x00 0x26 0x01>; + reg = <0x00 0xa002c00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a002e00 { + dma-coherent; + interrupts = <0x00 0x27 0x01>; + reg = <0x00 0xa002e00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003000 { + dma-coherent; + interrupts = <0x00 0x28 0x01>; + reg = <0x00 0xa003000 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003200 { + dma-coherent; + interrupts = <0x00 0x29 0x01>; + reg = <0x00 0xa003200 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003400 { + dma-coherent; + interrupts = <0x00 0x2a 0x01>; + reg = <0x00 0xa003400 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003600 { + dma-coherent; + interrupts = <0x00 0x2b 0x01>; + reg = <0x00 0xa003600 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003800 { + dma-coherent; + interrupts = <0x00 0x2c 0x01>; + reg = <0x00 0xa003800 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003a00 { + dma-coherent; + interrupts = <0x00 0x2d 0x01>; + reg = <0x00 0xa003a00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003c00 { + dma-coherent; + interrupts = <0x00 0x2e 0x01>; + reg = <0x00 0xa003c00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + virtio_mmio@a003e00 { + dma-coherent; + interrupts = <0x00 0x2f 0x01>; + reg = <0x00 0xa003e00 0x00 0x200>; + compatible = "virtio,mmio"; + }; + + gpio-keys { + compatible = "gpio-keys"; + + poweroff { + gpios = <0x8004 0x03 0x00>; + linux,code = <0x74>; + label = "GPIO Key Poweroff"; + }; + }; + + pl061@9030000 { + phandle = <0x8004>; + clock-names = "apb_pclk"; + clocks = <0x8000>; + interrupts = <0x00 0x07 0x04>; + gpio-controller; + #gpio-cells = <0x02>; + compatible = "arm,pl061\0arm,primecell"; + reg = <0x00 0x9030000 0x00 0x1000>; + }; + + pcie@10000000 { + interrupt-map-mask = <0x1800 0x00 0x00 0x07>; + interrupt-map = <0x00 0x00 0x00 0x01 0x8002 0x00 0x00 0x00 0x03 0x04 0x00 0x00 0x00 0x02 0x8002 0x00 0x00 0x00 0x04 0x04 0x00 0x00 0x00 0x03 0x8002 0x00 0x00 0x00 0x05 0x04 0x00 0x00 0x00 0x04 0x8002 0x00 0x00 0x00 0x06 0x04 0x800 0x00 0x00 0x01 0x8002 0x00 0x00 0x00 0x04 0x04 0x800 0x00 0x00 0x02 0x8002 0x00 0x00 0x00 0x05 0x04 0x800 0x00 0x00 0x03 0x8002 0x00 0x00 0x00 0x06 0x04 0x800 0x00 0x00 0x04 0x8002 0x00 0x00 0x00 0x03 0x04 0x1000 0x00 0x00 0x01 0x8002 0x00 0x00 0x00 0x05 0x04 0x1000 0x00 0x00 0x02 0x8002 0x00 0x00 0x00 0x06 0x04 0x1000 0x00 0x00 0x03 0x8002 0x00 0x00 0x00 0x03 0x04 0x1000 0x00 0x00 0x04 0x8002 0x00 0x00 0x00 0x04 0x04 0x1800 0x00 0x00 0x01 0x8002 0x00 0x00 0x00 0x06 0x04 0x1800 0x00 0x00 0x02 0x8002 0x00 0x00 0x00 0x03 0x04 0x1800 0x00 0x00 0x03 0x8002 0x00 0x00 0x00 0x04 0x04 0x1800 0x00 0x00 0x04 0x8002 0x00 0x00 0x00 0x05 0x04>; + #interrupt-cells = <0x01>; + ranges = <0x1000000 0x00 0x00 0x00 0x3eff0000 0x00 0x10000 0x2000000 0x00 0x10000000 0x00 0x10000000 0x00 0x2eff0000>; + reg = <0x00 0x3f000000 0x00 0x1000000>; + msi-map = <0x00 0x8003 0x00 0x10000>; + dma-coherent; + bus-range = <0x00 0x0f>; + linux,pci-domain = <0x00>; + #size-cells = <0x02>; + #address-cells = <0x03>; + device_type = "pci"; + compatible = "pci-host-ecam-generic"; + }; + + pl031@9010000 { + clock-names = "apb_pclk"; + clocks = <0x8000>; + interrupts = <0x00 0x02 0x04>; + reg = <0x00 0x9010000 0x00 0x1000>; + compatible = "arm,pl031\0arm,primecell"; + }; + + pl011@9000000 { + clock-names = "uartclk\0apb_pclk"; + clocks = <0x8000 0x8000>; + interrupts = <0x00 0x01 0x04>; + reg = <0x00 0x9000000 0x00 0x1000>; + compatible = "arm,pl011\0arm,primecell"; + }; + + pmu { + interrupts = <0x01 0x07 0x104>; + compatible = "arm,armv8-pmuv3"; + }; + + intc@8000000 { + phandle = <0x8002>; + interrupts = <0x01 0x09 0x04>; + reg = <0x00 0x8000000 0x00 0x10000 0x00 0x8010000 0x00 0x10000 0x00 0x8030000 0x00 0x10000 0x00 0x8040000 0x00 0x10000>; + compatible = "arm,cortex-a15-gic"; + ranges; + #size-cells = <0x02>; + #address-cells = <0x02>; + interrupt-controller; + #interrupt-cells = <0x03>; + + v2m@8020000 { + phandle = <0x8003>; + reg = <0x00 0x8020000 0x00 0x1000>; + msi-controller; + compatible = "arm,gic-v2m-frame"; + }; + }; + + flash@0 { + bank-width = <0x04>; + reg = <0x00 0x00 0x00 0x4000000 0x00 0x4000000 0x00 0x4000000>; + compatible = "cfi-flash"; + }; + + cpus { + #size-cells = <0x00>; + #address-cells = <0x01>; + + cpu-map { + + socket0 { + + cluster0 { + + core0 { + cpu = <0x8001>; + }; + }; + }; + }; + + cpu@0 { + phandle = <0x8001>; + reg = <0x00>; + compatible = "arm,cortex-a53"; + device_type = "cpu"; + }; + }; + + timer { + interrupts = <0x01 0x0d 0x104 0x01 0x0e 0x104 0x01 0x0b 0x104 0x01 0x0a 0x104>; + always-on; + compatible = "arm,armv8-timer\0arm,armv7-timer"; + }; + + apb-pclk { + phandle = <0x8000>; + clock-output-names = "clk24mhz"; + clock-frequency = <0x16e3600>; + #clock-cells = <0x00>; + compatible = "fixed-clock"; + }; + + chosen { + stdout-path = "/pl011@9000000"; + rng-seed = <0x39809366 0xe7fc0a9d 0x6c78703e 0xe7eba96c 0x94a9b477 0xfec74fb6 0xe647b61c 0xd80d1762>; + kaslr-seed = <0x2044fc98 0x9c6ba958>; + }; +}; From 077a5a8d6764d04c43fcf0751f1824773054f5b3 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Oct 2024 16:33:04 +1100 Subject: [PATCH 51/60] Document all the weird behaviours?!??! Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 460bf1b65..453ba40f6 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -171,7 +171,7 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) } } -void device_ack_irq(uint8_t bus, uint8_t device, uint8_t function, bool to_mask) +void print_pci_info(uint8_t bus, uint8_t device, uint8_t function, bool to_mask) { uintptr_t offset = get_bdf_offset(bus, device, function); assert(offset < PCIE_CONFIG_SIZE); @@ -186,6 +186,8 @@ void device_ack_irq(uint8_t bus, uint8_t device, uint8_t function, bool to_mask) sddf_dprintf("\t(PIN-based) interrupt disable: %s\n", (header->command & BIT(10)) ? "is disabled" : "is not disabled"); sddf_dprintf("\t(PIN-based) interrupt status: %s\n", (header->status & BIT(3)) ? "asserted" : "none"); + return; + /* mask out the interrupt... */ // https://elixir.bootlin.com/linux/v5.18-rc4/source/drivers/pci/pci.c#L4595 if (to_mask) { @@ -268,7 +270,7 @@ void init() #endif assert(found_nvme); - device_ack_irq(nvme_bus, nvme_device, nvme_function, false); + print_pci_info(nvme_bus, nvme_device, nvme_function, false); nvme_init(); } @@ -276,20 +278,37 @@ void nvme_continue(int z); void notified(microkit_channel ch) { static int i = 1; + sddf_dprintf("\n===============================\n"); sddf_dprintf("notified on ch: %u\n", ch); - if (i < 10) { - sddf_dprintf("acking (%u)\n", i); - // // device_print(nvme_bus, nvme_device, nvme_function); - // device_ack_irq(nvme_bus, nvme_device, nvme_function, false); - microkit_irq_ack(ch); + if (i <= 3 /* keep i nsync with nvme_continue */) { + /* this shows asserted on all platforms, as you would expect */ + print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); + sddf_dprintf("\nacking (%u)\n", i); + /* this usually acknowledges the CQ doorbell, causing interrupts to disappear + but for the second time we intentionally making the level interrupt fire + again by not acking it. + + on QEMU RISCV we see "should get an IRQ" and that's the last message + we see -> but [continues] + */ nvme_continue(i); - } else if (i < 20) { - sddf_dprintf("\nhopefully stopping\n"); + microkit_irq_ack(ch); + + /* on all platforms this shows interrupt status: none after the 3rd + ack when we stop doing anything. + + [...] on QEMU RISCV this shows "asserted" yet we don't get further IRQs + + AND!!! on Star64 RISCV this shows "none" but we still get interrupts!! + */ + print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); + } else if (i < 10) { + sddf_dprintf("\nYou should not see this message -- trying to ACK again\n"); microkit_irq_ack(ch); } else { - sddf_dprintf("stopping\n"); + sddf_dprintf("Stopping IRQ ACKing\n"); } i++; From 6a08e5b05b5c0718fa8e3e6fd6a6fa030cfc676d Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 18 Oct 2024 16:45:09 +1100 Subject: [PATCH 52/60] add TODO Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 453ba40f6..03836655d 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -294,6 +294,9 @@ void notified(microkit_channel ch) we see -> but [continues] */ nvme_continue(i); + /* TODO: we should use NVMe INTMS/INTMC while this is happening + because that PCI virtual INTx line is shared + howveer since we don't share it, meh! */ microkit_irq_ack(ch); /* on all platforms this shows interrupt status: none after the 3rd From d0eed1ac9b4aa5b5c2ad7645324d9ede29a52903 Mon Sep 17 00:00:00 2001 From: julia Date: Mon, 28 Oct 2024 16:02:02 +1100 Subject: [PATCH 53/60] fix IRQ comment Signed-off-by: julia --- examples/pcie/board/star64/pcie.system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pcie/board/star64/pcie.system b/examples/pcie/board/star64/pcie.system index df32bd803..dbf5f6c62 100644 --- a/examples/pcie/board/star64/pcie.system +++ b/examples/pcie/board/star64/pcie.system @@ -22,7 +22,7 @@ - + From 6639e0cf6e8d65bac48432b21e8d2acc7bcc9f3b Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 29 Oct 2024 10:39:15 +1100 Subject: [PATCH 54/60] set microkit sdk env var in zig build Signed-off-by: julia --- examples/pcie/build.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index 8fcca94d7..bc2b949cd 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -128,6 +128,7 @@ pub fn build(b: *std.Build) void { const system_description_path = b.fmt("board/{s}/pcie.system", .{microkit_board}); const final_image_dest = b.getInstallPath(.bin, "./loader.img"); const microkit_tool_cmd = b.addSystemCommand(&[_][]const u8{ microkit_tool, system_description_path, "--search-path", b.getInstallPath(.bin, ""), "--board", microkit_board, "--config", microkit_config, "-o", final_image_dest, "-r", b.getInstallPath(.prefix, "./report.txt") }); + microkit_tool_cmd.setEnvironmentVariable("MICROKIT_SDK", microkit_sdk); microkit_tool_cmd.step.dependOn(b.getInstallStep()); microkit_tool_cmd.step.dependOn(&driver_install.step); const microkit_step = b.step("microkit", "Compile and build the final bootable image"); From a705d96034330e15c1dcfb43e7b4db6edd1d23ac Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 29 Oct 2024 13:14:56 +1100 Subject: [PATCH 55/60] mostly-broken rockpro Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 23 ++++++++++- examples/pcie/board/rockpro64/pcie.system | 47 +++++++++++++++++++++++ examples/pcie/build.zig | 11 ++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 examples/pcie/board/rockpro64/pcie.system diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 03836655d..6ead09eeb 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -19,7 +19,12 @@ void nvme_init(); See [PCIe-2.0] §7.22 */ uintptr_t pcie_config; +// TODO: different for most platforms +#if defined(CONFIG_PLAT_ROCKPRO64) +#define PCIE_CONFIG_SIZE (0x1000000 - 0x800000) +#else #define PCIE_CONFIG_SIZE 0x1000000 +#endif /* bus between [0, 256) device between [0, 31) @@ -53,6 +58,14 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) return; } +#if defined(CONFIG_PLAT_ROCKPRO64) + // https://github.com/torvalds/linux/commit/d84c572de1a360501d2e439ac632126f5facf59d + if (bus == 0 && device > 0) { + return; + } +#endif + +#if defined(CONFIG_PLAT_STAR64) /* See: plda_pcie_addr_valid() in U-Boot https://lore.kernel.org/u-boot/20230423105859.125764-2-minda.chen@starfivetech.com/ @@ -67,16 +80,22 @@ void device_print(uint8_t bus, uint8_t device, uint8_t function) if (bus == 1 && device > 0) { return; } +#endif sddf_dprintf("\nB.D:F: %02x:%02x.%01x\n", bus, device, function); - // enable bus mastering... and memory accesses - header->command |= BIT(2) | BIT(1); sddf_dprintf("vendor ID: 0x%04x\n", header->vendor_id); sddf_dprintf("device ID: 0x%04x\n", header->device_id); sddf_dprintf("command register: 0x%04x\n", header->command); sddf_dprintf("status register: 0x%04x\n", header->status); sddf_dprintf("revision ID: 0x%02x\n", header->revision_id); sddf_dprintf("base-class code: 0x%02x | sub-class code: 0x%02x\n", header->base_class_code, header->subclass_code); + + // rockchip is ????? +#if !defined(CONFIG_PLAT_ROCKPRO64) + // enable bus mastering... and memory accesses + header->command |= BIT(2) | BIT(1); +#endif + sddf_dprintf("enabled bus master and memory access\n"); if (header->base_class_code == 0x1 && header->subclass_code == 0x8 && !found_nvme) { sddf_dprintf("FOUND NVME!!!\n"); found_nvme = true; diff --git a/examples/pcie/board/rockpro64/pcie.system b/examples/pcie/board/rockpro64/pcie.system new file mode 100644 index 000000000..613612478 --- /dev/null +++ b/examples/pcie/board/rockpro64/pcie.system @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index bc2b949cd..06305e3c2 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -6,6 +6,7 @@ const std = @import("std"); const MicrokitBoard = enum { star64, + rockpro64, qemu_virt_aarch64, qemu_virt_riscv64, }; @@ -25,6 +26,15 @@ const targets = [_]Target{ .abi = .none, }, }, + .{ + .board = MicrokitBoard.rockpro64, + .zig_target = std.Target.Query{ + .cpu_arch = .aarch64, + .cpu_model = .{ .explicit = &std.Target.arm.cpu.cortex_a53 }, + .os_tag = .freestanding, + .abi = .none, + }, + }, .{ .board = MicrokitBoard.qemu_virt_aarch64, .zig_target = std.Target.Query{ @@ -98,6 +108,7 @@ pub fn build(b: *std.Build) void { const driver_class = switch (microkit_board_option.?) { .star64 => "starfive", + .rockpro64 => "starfive", // hack .qemu_virt_aarch64 => "starfive", // hack .qemu_virt_riscv64 => "starfive", // hack }; From a688a98b2fab119c71440702c9849899e5823cc3 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 29 Oct 2024 15:18:00 +1100 Subject: [PATCH 56/60] nvme boots ish on rockpro Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 10 +++++----- examples/pcie/build.zig | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 6ead09eeb..8a0f757b9 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -268,7 +268,7 @@ void init() { sddf_dprintf("pcie driver starting!\n"); -#if 1 +#if 0 for (uint8_t bus = 0; bus <= 255; bus++) { for (uint8_t device = 0; device < 32; device++) { for (uint8_t function = 0; function < 8; function++) { @@ -288,8 +288,8 @@ void init() sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); #endif - assert(found_nvme); - print_pci_info(nvme_bus, nvme_device, nvme_function, false); + // assert(found_nvme); + // print_pci_info(nvme_bus, nvme_device, nvme_function, false); nvme_init(); } @@ -302,7 +302,7 @@ void notified(microkit_channel ch) if (i <= 3 /* keep i nsync with nvme_continue */) { /* this shows asserted on all platforms, as you would expect */ - print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); + // print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); sddf_dprintf("\nacking (%u)\n", i); /* this usually acknowledges the CQ doorbell, causing interrupts to disappear @@ -325,7 +325,7 @@ void notified(microkit_channel ch) AND!!! on Star64 RISCV this shows "none" but we still get interrupts!! */ - print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); + // print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); } else if (i < 10) { sddf_dprintf("\nYou should not see this message -- trying to ACK again\n"); microkit_irq_ack(ch); diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index 06305e3c2..a977c2a7f 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -30,7 +30,8 @@ const targets = [_]Target{ .board = MicrokitBoard.rockpro64, .zig_target = std.Target.Query{ .cpu_arch = .aarch64, - .cpu_model = .{ .explicit = &std.Target.arm.cpu.cortex_a53 }, + .cpu_model = .{ .explicit = &std.Target.aarch64.cpu.cortex_a53 }, + .cpu_features_add = std.Target.aarch64.featureSet(&[_]std.Target.aarch64.Feature{ .strict_align }), .os_tag = .freestanding, .abi = .none, }, From 78411f7072f613340919bdb292a17538d9548088 Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 5 Nov 2024 17:24:31 +1100 Subject: [PATCH 57/60] imx8mm pcie evk worky Signed-off-by: julia --- drivers/pcie/starfive/pcie.c | 20 ++++---- examples/pcie/board/imx8mm_evk/pcie.system | 54 ++++++++++++++++++++++ examples/pcie/build.zig | 20 ++++++-- 3 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 examples/pcie/board/imx8mm_evk/pcie.system diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 8a0f757b9..54175a3ff 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -19,11 +19,14 @@ void nvme_init(); See [PCIe-2.0] §7.22 */ uintptr_t pcie_config; -// TODO: different for most platforms #if defined(CONFIG_PLAT_ROCKPRO64) #define PCIE_CONFIG_SIZE (0x1000000 - 0x800000) -#else +#elif defined(CONFIG_PLAT_IMX8MM_EVK) +#define PCIE_CONFIG_SIZE 0x10000 +#elif defined(CONFIG_PLAT_STAR64) #define PCIE_CONFIG_SIZE 0x1000000 +#else +#error "unknown pcie config region size" #endif /* bus between [0, 256) @@ -268,10 +271,11 @@ void init() { sddf_dprintf("pcie driver starting!\n"); -#if 0 +#if 1 for (uint8_t bus = 0; bus <= 255; bus++) { for (uint8_t device = 0; device < 32; device++) { - for (uint8_t function = 0; function < 8; function++) { + // for (uint8_t function = 0; function < 8; function++) { + for (uint8_t function = 0; function < 1; function++) { uintptr_t offset = get_bdf_offset(bus, device, function); if (offset >= PCIE_CONFIG_SIZE) { goto out; @@ -288,8 +292,8 @@ void init() sddf_dprintf("\n\nPCIE_ENUM_COMPLETE\n"); #endif - // assert(found_nvme); - // print_pci_info(nvme_bus, nvme_device, nvme_function, false); + assert(found_nvme); + print_pci_info(nvme_bus, nvme_device, nvme_function, false); nvme_init(); } @@ -302,7 +306,7 @@ void notified(microkit_channel ch) if (i <= 3 /* keep i nsync with nvme_continue */) { /* this shows asserted on all platforms, as you would expect */ - // print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); + print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); sddf_dprintf("\nacking (%u)\n", i); /* this usually acknowledges the CQ doorbell, causing interrupts to disappear @@ -325,7 +329,7 @@ void notified(microkit_channel ch) AND!!! on Star64 RISCV this shows "none" but we still get interrupts!! */ - // print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); + print_pci_info(nvme_bus, nvme_device, nvme_function, /* don't ack? */ false); } else if (i < 10) { sddf_dprintf("\nYou should not see this message -- trying to ACK again\n"); microkit_irq_ack(ch); diff --git a/examples/pcie/board/imx8mm_evk/pcie.system b/examples/pcie/board/imx8mm_evk/pcie.system new file mode 100644 index 000000000..9197a902b --- /dev/null +++ b/examples/pcie/board/imx8mm_evk/pcie.system @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/pcie/build.zig b/examples/pcie/build.zig index a977c2a7f..46fd17fb9 100644 --- a/examples/pcie/build.zig +++ b/examples/pcie/build.zig @@ -5,10 +5,11 @@ const std = @import("std"); const MicrokitBoard = enum { - star64, - rockpro64, - qemu_virt_aarch64, - qemu_virt_riscv64, + star64, // interrupts dodgy + rockpro64, // not working + imx8mm_evk, + qemu_virt_aarch64, // works + qemu_virt_riscv64, // interrupts dodgy }; const Target = struct { @@ -31,6 +32,16 @@ const targets = [_]Target{ .zig_target = std.Target.Query{ .cpu_arch = .aarch64, .cpu_model = .{ .explicit = &std.Target.aarch64.cpu.cortex_a53 }, +// .cpu_features_add = std.Target.aarch64.featureSet(&[_]std.Target.aarch64.Feature{ .strict_align }), + .os_tag = .freestanding, + .abi = .none, + }, + }, + .{ + .board = MicrokitBoard.imx8mm_evk, + .zig_target = std.Target.Query{ + .cpu_arch = .aarch64, + .cpu_model = .{ .explicit = &std.Target.aarch64.cpu.cortex_a53 }, .cpu_features_add = std.Target.aarch64.featureSet(&[_]std.Target.aarch64.Feature{ .strict_align }), .os_tag = .freestanding, .abi = .none, @@ -110,6 +121,7 @@ pub fn build(b: *std.Build) void { const driver_class = switch (microkit_board_option.?) { .star64 => "starfive", .rockpro64 => "starfive", // hack + .imx8mm_evk => "starfive", // hack .qemu_virt_aarch64 => "starfive", // hack .qemu_virt_riscv64 => "starfive", // hack }; From 0b0490b42e4c822430229d56d5d944aaba32b217 Mon Sep 17 00:00:00 2001 From: julia Date: Thu, 7 Nov 2024 22:56:42 +1100 Subject: [PATCH 58/60] working nvme on the imx8 --- examples/pcie/board/imx8mm_evk/pcie.system | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/pcie/board/imx8mm_evk/pcie.system b/examples/pcie/board/imx8mm_evk/pcie.system index 9197a902b..cd4fe8c28 100644 --- a/examples/pcie/board/imx8mm_evk/pcie.system +++ b/examples/pcie/board/imx8mm_evk/pcie.system @@ -12,7 +12,8 @@ - + + From a41d9518e647bc60015b10986a9b0baa2e6dc786 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 8 Nov 2024 12:55:11 +1100 Subject: [PATCH 59/60] qemu riscv pci size --- drivers/pcie/starfive/pcie.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 54175a3ff..03758b7fc 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -25,6 +25,8 @@ uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x10000 #elif defined(CONFIG_PLAT_STAR64) #define PCIE_CONFIG_SIZE 0x1000000 +#elif defined(CONFIG_PLAT_QEMU_RISCV_VIRT) +#define PCIE_CONFIG_SIZE 0x1000000 #else #error "unknown pcie config region size" #endif From b14f3fbe5d2d1602de6b78a5230ac0b45594876e Mon Sep 17 00:00:00 2001 From: julia Date: Tue, 12 Nov 2024 14:35:55 +1100 Subject: [PATCH 60/60] config size of qemu aarch64 --- drivers/pcie/starfive/pcie.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pcie/starfive/pcie.c b/drivers/pcie/starfive/pcie.c index 03758b7fc..4a518f539 100644 --- a/drivers/pcie/starfive/pcie.c +++ b/drivers/pcie/starfive/pcie.c @@ -25,7 +25,7 @@ uintptr_t pcie_config; #define PCIE_CONFIG_SIZE 0x10000 #elif defined(CONFIG_PLAT_STAR64) #define PCIE_CONFIG_SIZE 0x1000000 -#elif defined(CONFIG_PLAT_QEMU_RISCV_VIRT) +#elif defined(CONFIG_PLAT_QEMU_RISCV_VIRT) || defined(CONFIG_PLAT_QEMU_ARM_VIRT) #define PCIE_CONFIG_SIZE 0x1000000 #else #error "unknown pcie config region size"