Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Summary: creating single boot ROM for multicore configs #80

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Summary: creating single boot ROM for multicore configs #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion include/riscv_cpu.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@

#define ROM_SIZE 0x00002000
#define ROM_BASE_ADDR 0x00010000
#define BOOT_BASE_ADDR 0x00010000
#define BOOT_BASE_ADDR 0x00010040

// The default RAM base, can be relocated with config "memory_base_addr"
#define RAM_BASE_ADDR 0x80000000
Expand Down Expand Up @@ -363,6 +363,8 @@ void riscv_set_debug_mode(RISCVCPUState *s, bool on);
int riscv_benchmark_exit_code(RISCVCPUState *s);

#include "riscv_machine.h"
void generate_core_boot_rom(uint32_t *rom, uint32_t rom_size, uint32_t code_pos, uint32_t data_pos, RISCVCPUState *s, const uint64_t clint_base_addr);
void create_boot_rom_image(uint32_t *rom, uint32_t rom_size_bytes, const char *file_name);
void riscv_ram_serialize(RISCVCPUState *s, const char *dump_name);
void riscv_cpu_serialize(RISCVCPUState *s, const char *dump_name, const uint64_t clint_base_addr);
void riscv_ram_deserialize(RISCVCPUState *s, const char *dump_name);
Expand Down
2 changes: 1 addition & 1 deletion src/dromajo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ int main(int argc, char **argv) {
if (!m)
return 1;

int n_cycles = 10000;
int n_cycles = 1;
execution_start_ts = get_current_time_in_seconds();
execution_progress_meassure = &m->cpu_state[0]->minstret;
signal(SIGINT, sigintr_handler);
Expand Down
79 changes: 25 additions & 54 deletions src/riscv_cpu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2354,10 +2354,23 @@ static void deserialize_memory(void *base, size_t size, const char *file) {
if (f_fd < 0)
err(-3, "trying to read %s", file);

size_t sz = read(f_fd, base, size);
size_t read_size = 0;
uint8_t *ptr = (uint8_t *)base;
int64_t pending_size = size;
do {
/* Linux reads in 2GB chunks at most. */
size_t sz = read(f_fd, &ptr[read_size], pending_size);
if (sz <= 0) {
err(-3, "%s %zd size failed to read memory size %zd", file, sz, size);
break;
}

read_size += sz;
pending_size -= sz;
} while(pending_size > 0);

if (sz != size)
err(-3, "%s %zd size does not match memory size %zd", file, sz, size);
if (read_size != size)
err(-3, "%s %zd size does not match memory size %zd", file, read_size, size);

close(f_fd);
}
Expand Down Expand Up @@ -2523,22 +2536,10 @@ static void create_hang_nonzero_hart(uint32_t *rom, uint32_t *code_pos, uint32_t
// 1:
}

static void create_boot_rom(RISCVCPUState *s, const char *file, const uint64_t clint_base_addr) {
uint32_t rom[ROM_SIZE / 4];
memset(rom, 0, sizeof rom);

// ROM organization
// 0000..003F wasted
// 0040..0AFF boot code (2,752 B)
// 0B00..0FFF boot data ( 512 B)

uint32_t code_pos = (BOOT_BASE_ADDR - ROM_BASE_ADDR) / sizeof *rom;
uint32_t data_pos = 0xB00 / sizeof *rom;
void generate_core_boot_rom(uint32_t *rom, uint32_t rom_size, uint32_t code_pos, uint32_t data_pos, RISCVCPUState *s, const uint64_t clint_base_addr) {
/* Remember the start position for boundry checks. */
uint32_t data_pos_start = data_pos;

if (s->machine->ncpus == 1) // FIXME: May be interesting to freeze hartid >= ncpus
create_hang_nonzero_hart(rom, &code_pos, &data_pos);

create_csr64_recovery(rom, &code_pos, &data_pos, 0x7b1, s->pc); // Write to DPC (CSR, 0x7b1)

// Write current priviliege level to prv in dcsr (0 user, 1 supervisor, 2 user)
Expand Down Expand Up @@ -2668,7 +2669,7 @@ static void create_boot_rom(RISCVCPUState *s, const char *file, const uint64_t c
// dret 0x7b200073
rom[code_pos++] = 0x7b200073;

if (sizeof rom / sizeof *rom <= data_pos || data_pos_start <= code_pos) {
if (rom_size <= data_pos || data_pos_start <= code_pos) {
fprintf(dromajo_stderr,
"ERROR: ROM is too small. ROM_SIZE should increase. "
"Current code_pos=%d data_pos=%d\n",
Expand All @@ -2677,7 +2678,11 @@ static void create_boot_rom(RISCVCPUState *s, const char *file, const uint64_t c
exit(-6);
}

serialize_memory(rom, ROM_SIZE, file);
}

void create_boot_rom_image(uint32_t *rom, uint32_t rom_size_bytes, const char *file_name) {
// Write ROM.
serialize_memory(rom, rom_size_bytes, file_name);
}

void riscv_ram_serialize(RISCVCPUState *s, const char *dump_name) {
Expand Down Expand Up @@ -2759,40 +2764,6 @@ void riscv_cpu_serialize(RISCVCPUState *s, const char *dump_name, const uint64_t

for (int i = 0; i < 4; i += 2) fprintf(conf_fd, "pmpcfg%d:%llx\n", i, (unsigned long long)s->csr_pmpcfg[i]);
for (int i = 0; i < 16; ++i) fprintf(conf_fd, "pmpaddr%d:%llx\n", i, (unsigned long long)s->csr_pmpaddr[i]);

PhysMemoryRange *boot_ram = 0;
for (int i = s->mem_map->n_phys_mem_range - 1; i >= 0; --i) {
PhysMemoryRange *pr = &s->mem_map->phys_mem_range[i];
fprintf(conf_fd, "mrange%d:0x%llx 0x%llx %s\n", i, (long long)pr->addr, (long long)pr->size, pr->is_ram ? "ram" : "io");

if (pr->is_ram && pr->addr == ROM_BASE_ADDR) {
assert(!boot_ram);
boot_ram = pr;
}
}

if (!boot_ram) {
fprintf(dromajo_stderr, "ERROR: could not find boot RAM.\n");
exit(-3);
}

n = strlen(dump_name) + 64;
char *f_name = (char *)alloca(n);
snprintf(f_name, n, "%s.bootram", dump_name);

if (s->priv != 3 || ROM_BASE_ADDR + ROM_SIZE < s->pc) {
fprintf(dromajo_stderr, "NOTE: creating a new boot ROM.\n");
create_boot_rom(s, f_name, clint_base_addr);
} else if (BOOT_BASE_ADDR < s->pc) {
fprintf(dromajo_stderr, "ERROR: could not checkpoint when running inside the ROM.\n");
exit(-4);
} else if (s->pc == BOOT_BASE_ADDR && boot_ram) {
fprintf(dromajo_stderr, "NOTE: using the default dromajo ROM.\n");
serialize_memory(boot_ram->phys_mem, boot_ram->size, f_name);
} else {
fprintf(dromajo_stderr, "ERROR: unexpected PC address 0x%llx.\n", (long long)s->pc);
exit(-4);
}
}

void riscv_ram_deserialize(RISCVCPUState *s, const char *dump_name) {
Expand All @@ -2818,6 +2789,6 @@ void riscv_cpu_deserialize(RISCVCPUState *s, const char *dump_name) {
snprintf(boot_name, n, "%s.bootram", dump_name);

deserialize_memory(pr->phys_mem, pr->size, boot_name);
}
}
}
}
91 changes: 73 additions & 18 deletions src/riscv_machine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1199,7 +1199,9 @@ RISCVMachine *virt_machine_init(const VirtMachineParams *p) {

/* RAM */
cpu_register_ram(s->mem_map, s->ram_base_addr, s->ram_size, 0);
cpu_register_ram(s->mem_map, ROM_BASE_ADDR, ROM_SIZE, 0);

/* Boot ROM. */
cpu_register_ram(s->mem_map, ROM_BASE_ADDR, s->ncpus * ROM_SIZE, 0);

for (int i = 0; i < s->ncpus; ++i) {
s->cpu_state[i]->physical_addr_len = p->physical_addr_len;
Expand Down Expand Up @@ -1416,32 +1418,85 @@ void virt_machine_end(RISCVMachine *s) {
}

void virt_machine_serialize(RISCVMachine *m, const char *dump_name) {
/* Serialize core states. */
for (int i = 0; i < m->ncpus; ++i) {
/* Check that all cores are out of the ROM. */
bool is_serializable = true;
for (int i = 0; i < m->ncpus && is_serializable; ++i) {
RISCVCPUState *s = m->cpu_state[i];
is_serializable = s->priv != 3 || (ROM_BASE_ADDR + (m->ncpus * ROM_SIZE) < s->pc);
}

vm_error("plic: %x %x timecmp=%llx\n", m->plic_pending_irq, m->plic_served_irq, (unsigned long long)s->timecmp);
/* Serialize core states. */
if (is_serializable) {
/* Create serialization file per core. */
for (int i = 0; i < m->ncpus; ++i) {
RISCVCPUState *s = m->cpu_state[i];

/* Append core number suffix to the file name. */
std::stringstream core_dump_name;
core_dump_name << dump_name << i;
riscv_cpu_serialize(s, core_dump_name.str().c_str(), m->clint_base_addr);
}
vm_error("plic: %x %x timecmp=%llx\n", m->plic_pending_irq, m->plic_served_irq, (unsigned long long)s->timecmp);

/* Serialize memory. */
riscv_ram_serialize(m->cpu_state[0], dump_name);
/* Append core number suffix to the file name. */
std::stringstream core_dump_name;
core_dump_name << dump_name << i;

riscv_cpu_serialize(s, core_dump_name.str().c_str(), m->clint_base_addr);
}

/* Generate single boot ROM for all cores. */
const uint32_t kTotalRomSize = (m->ncpus * ROM_SIZE) / 4;
uint32_t rom[kTotalRomSize];
memset(rom, 0, sizeof(rom));

// ROM organization
// Core 0:
// 0000..003F wasted
// 0040..0AFF boot code (2,752 B)
// 0B00..0FFF boot data ( 512 B)
// Core 1:
// 1000..003F wasted
// 1040..0AFF boot code (2,752 B)
// 1B00..0FFF boot data ( 512 B)
// repeats for each core ...
for (int i = 0; i < m->ncpus; ++i) {
RISCVCPUState *s = m->cpu_state[i];
uint32_t code_pos = ((i << 12) | (BOOT_BASE_ADDR - ROM_BASE_ADDR)) / sizeof(*rom);
uint32_t data_pos = ((i << 12) | 0xB00) / sizeof(*rom);

/* All cores start by determining the PC they should jump to. */
rom[code_pos++] = 0xf1402573; // csrr a0, mhartid
rom[code_pos++] = 0x00c5151b; // slliw a0, a0, 12

/* These four instructions must be the last in preamble. */
/* If other instructions should be added, add them before these four. */
rom[code_pos++] = 0x00000597; // auipc a1, 0x0
rom[code_pos++] = 0x0105859b; // addiw a1, a1, 0xc
rom[code_pos++] = 0x00b5053b; // addw a0, a0, a1
rom[code_pos++] = 0x50067; // jr a0

/* Generates and appends recovery code for each core to rom. */
generate_core_boot_rom(rom, kTotalRomSize, code_pos, data_pos, s, m->clint_base_addr);
}

/* Write generated boot ROM to file. */
uint32_t name_len = strlen(dump_name) + 64;
char *f_name = (char *)alloca(name_len);
snprintf(f_name, name_len, "%s.bootram", dump_name);
create_boot_rom_image(rom, 4 * kTotalRomSize, f_name);

/* Write memory state. */
riscv_ram_serialize(m->cpu_state[0], dump_name);
}
else
{
fprintf(dromajo_stderr, "ERROR: could not checkpoint. One or more cores running inside the ROM.\n");
exit(-4);
}
}

void virt_machine_deserialize(RISCVMachine *m, const char *dump_name) {
/* Deserialize core states. */
for (int i = 0; i < m->ncpus; ++i) {
RISCVCPUState *s = m->cpu_state[i];
RISCVCPUState *s = m->cpu_state[0];

/* Append core number suffix to the file name. */
std::stringstream core_dump_name;
core_dump_name << dump_name << i;
riscv_cpu_deserialize(s, core_dump_name.str().c_str());
}
/* Append core number suffix to the file name. */
riscv_cpu_deserialize(s, dump_name);

/* Deserialize memory. */
riscv_ram_deserialize(m->cpu_state[0], dump_name);
Expand Down