Skip to content

Commit

Permalink
Add support for multiple vCPUs
Browse files Browse the repository at this point in the history
Signed-off-by: Ivan Velickovic <[email protected]>
  • Loading branch information
Ivan-Velickovic committed Aug 19, 2024
1 parent 960a55d commit 7e4f2b9
Show file tree
Hide file tree
Showing 11 changed files with 133 additions and 52 deletions.
19 changes: 12 additions & 7 deletions include/libvmm/arch/aarch64/vgic/vdist.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* SPDX-License-Identifier: BSD-2-Clause
*/

#include <libvmm/vcpu.h>
#include <libvmm/util/util.h>
#include <libvmm/arch/aarch64/fault.h>

Expand Down Expand Up @@ -494,8 +495,8 @@ static bool vgic_dist_reg_write(size_t vcpu_id, vgic_t *vgic, uint64_t offset, u
irq = CTZ(data);
data &= ~(1U << irq);
irq += (offset - GIC_DIST_ISPENDR0) * 8;
// @ivanv: should be checking this and other calls like it succeed
vgic_dist_set_pending_irq(vgic, vcpu_id, irq);
success = vgic_dist_set_pending_irq(vgic, vcpu_id, irq);
assert(success);
}
break;
case RANGE32(GIC_DIST_ICPENDR0, GIC_DIST_ICPENDRN):
Expand Down Expand Up @@ -571,11 +572,15 @@ static bool vgic_dist_reg_write(size_t vcpu_id, vgic_t *vgic, uint64_t offset, u
LOG_VMM_ERR("Unknown SGIR Target List Filter mode");
goto ignore_fault;
}
// @ivanv: Here we're making the assumption that there's only one vCPU, and
// we're also blindly injectnig the given IRQ to that vCPU.
// @ivanv: come back to this, do we have two writes to the TCB registers?
success = vgic_inject_irq(vcpu_id, virq);
assert(success);
for (int i = 0; i < GUEST_NUM_VCPUS; i++) {
if ((1 << i) & target_list && vcpu_is_on(i)) {
success = vgic_inject_irq(i, virq);
assert(success);
if (!success) {
return false;
}
}
}
break;
case RANGE32(0xF04, 0xF0C):
/* Reserved */
Expand Down
2 changes: 2 additions & 0 deletions include/libvmm/util/util.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
#define GUEST_VCPU_ID 0
#define GUEST_NUM_VCPUS 1

#define PAGE_SIZE_MIN 0x1000

// @ivanv: if we keep using this, make sure that we have a static assert
// that sizeof seL4_UserContext is 0x24
// Note that this is AArch64 specific
Expand Down
15 changes: 15 additions & 0 deletions include/libvmm/vcpu.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,22 @@

#pragma once

#include <stdbool.h>
#include <stddef.h>

/*
* Checks whether the given vCPU is on.
*
* Returns true if the vCPU is on, otherwise false.
* Returns false if an invalid vCPU ID is given.
*/
bool vcpu_is_on(size_t vcpu_id);

/*
* Change the state tracking for whether a given vCPU is on.
* Does nothing if an invalid vCPU ID is given.
*/
void vcpu_set_on(size_t vcpu_id, bool on);

void vcpu_reset(size_t vcpu_id);
void vcpu_print_regs(size_t vcpu_id);
7 changes: 3 additions & 4 deletions src/arch/aarch64/fault.c
Original file line number Diff line number Diff line change
Expand Up @@ -251,11 +251,11 @@ bool fault_handle_user_exception(size_t vcpu_id)
// @ivanv: print out VM name/vCPU id when we have multiple VMs
size_t fault_ip = microkit_mr_get(seL4_UserException_FaultIP);
size_t number = microkit_mr_get(seL4_UserException_Number);
LOG_VMM_ERR("Invalid instruction fault at IP: 0x%lx, number: 0x%lx", fault_ip, number);
LOG_VMM_ERR("Invalid instruction fault at IP: 0x%lx, number: 0x%lx from vCPU %d\n", fault_ip, number, vcpu_id);
/* All we do is dump the TCB registers. */
tcb_print_regs(vcpu_id);

return true;
return false;
}

// @ivanv: document where these come from
Expand Down Expand Up @@ -312,8 +312,7 @@ bool fault_register_vm_exception_handler(uintptr_t base, size_t size, vm_excepti
return false;
}

// @ivanv: use a define for page size? preMAture GENeraliZAATION
if (base % 0x1000 != 0) {
if (base % PAGE_SIZE_MIN != 0) {
return false;
}

Expand Down
45 changes: 38 additions & 7 deletions src/arch/aarch64/psci.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#include <stdbool.h>
#include <libvmm/guest.h>
#include <libvmm/vcpu.h>
#include <libvmm/util/util.h>
#include <libvmm/arch/aarch64/psci.h>
#include <libvmm/arch/aarch64/smc.h>
Expand Down Expand Up @@ -44,13 +45,43 @@ bool handle_psci(size_t vcpu_id, seL4_UserContext *regs, uint64_t fn_number, uin
break;
}
case PSCI_CPU_ON: {
uintptr_t target_cpu = smc_get_arg(regs, 1);
// Right now we only have one vCPU and so any fault for a target vCPU
// that isn't the one that's already on we consider an error on the
// guest's side.
// @ivanv: adapt for starting other vCPUs
if (target_cpu == vcpu_id) {
smc_set_return_value(regs, PSCI_ALREADY_ON);
size_t target_vcpu = smc_get_arg(regs, 1);
if (target_vcpu < GUEST_NUM_VCPUS && target_vcpu >= 0) {
/* The guest has given a valid target vCPU */
if (vcpu_is_on(target_vcpu)) {
smc_set_return_value(regs, PSCI_ALREADY_ON);
} else {
/* We have a valid target vCPU, that is not started yet. So let's turn it on. */
uintptr_t vcpu_entry_point = smc_get_arg(regs, 2);
size_t context_id = smc_get_arg(regs, 3);

seL4_UserContext vcpu_regs = {0};
vcpu_regs.x0 = context_id;
vcpu_regs.spsr = 5; // PMODE_EL1h
vcpu_regs.pc = vcpu_entry_point;

microkit_vcpu_arm_write_reg(target_vcpu, seL4_VCPUReg_VMPIDR_EL2, target_vcpu);

seL4_Error err = seL4_TCB_WriteRegisters(
BASE_VM_TCB_CAP + target_vcpu,
false, // We'll explcitly start the guest below rather than in this call
0, // No flags
SEL4_USER_CONTEXT_SIZE,
&vcpu_regs
);
assert(err == seL4_NoError);
if (err != seL4_NoError) {
return err;
}

/* Now that we have started the vCPU, we can set is as turned on. */
vcpu_set_on(target_vcpu, true);

LOG_VMM("starting guest vCPU (0x%lx) with entry point 0x%lx, context ID: 0x%lx\n", target_vcpu, vcpu_regs.pc, context_id);
microkit_vcpu_restart(target_vcpu, vcpu_regs.pc);

smc_set_return_value(regs, PSCI_SUCCESS);
}
} else {
// The guest has requested to turn on a virtual CPU that does
// not exist.
Expand Down
20 changes: 20 additions & 0 deletions src/arch/aarch64/vcpu.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,26 @@
#define SCTLR_EL1_NATIVE (SCTLR_EL1 | SCTLR_EL1_C | SCTLR_EL1_I | SCTLR_EL1_UCI)
#define SCTLR_DEFAULT SCTLR_EL1_NATIVE

bool vcpu_on_state[GUEST_NUM_VCPUS];

bool vcpu_is_on(size_t vcpu_id) {
assert(vcpu_id < GUEST_NUM_VCPUS);
if (vcpu_id >= GUEST_NUM_VCPUS) {
return false;
}

return vcpu_on_state[vcpu_id];
}

void vcpu_set_on(size_t vcpu_id, bool on) {
assert(vcpu_id < GUEST_NUM_VCPUS);
if (vcpu_id >= GUEST_NUM_VCPUS) {
return;
}

vcpu_on_state[vcpu_id] = on;
}

void vcpu_reset(size_t vcpu_id) {
// @ivanv this is an incredible amount of system calls
// Reset registers
Expand Down
7 changes: 5 additions & 2 deletions src/arch/aarch64/vgic/vgic.c
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <microkit.h>
#include <libvmm/vcpu.h>
#include <libvmm/util/util.h>
#include <libvmm/arch/aarch64/fault.h>
#include <libvmm/arch/aarch64/vgic/vgic.h>
Expand Down Expand Up @@ -83,9 +84,10 @@ bool vgic_register_irq(size_t vcpu_id, int virq_num, virq_ack_fn_t ack_fn, void

bool vgic_inject_irq(size_t vcpu_id, int irq)
{
LOG_IRQ("Injecting IRQ %d\n", irq);
LOG_IRQ("(vCPU %d) injecting IRQ %d\n", vcpu_id, irq);

return vgic_dist_set_pending_irq(&vgic, vcpu_id, irq);
bool success = vgic_dist_set_pending_irq(&vgic, vcpu_id, irq);
assert(success);

// @ivanv: explain why we don't check error before checking this fault stuff
// @ivanv: seperately, it seems weird to have this fault handling code here?
Expand All @@ -94,6 +96,7 @@ bool vgic_inject_irq(size_t vcpu_id, int irq)
// // ignore_fault(vcpu->vcpu_arch.fault);
// err = advance_vcpu_fault(regs);
// }
return success;
}

// @ivanv: revisit this whole function
Expand Down
37 changes: 19 additions & 18 deletions src/arch/aarch64/vgic/vgic_v2.c
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ static void vgic_dist_reset(struct gic_dist_map *gic_dist)
gic_dist->typer = 0x0000fce7; /* RO */
gic_dist->iidr = 0x0200043b; /* RO */

for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) {
for (int i = 0; i < GUEST_NUM_VCPUS; i++) {
gic_dist->enable_set0[i] = 0x0000ffff; /* 16bit RO */
gic_dist->enable_clr0[i] = 0x0000ffff; /* 16bit RO */
}
Expand All @@ -83,7 +83,7 @@ static void vgic_dist_reset(struct gic_dist_map *gic_dist)
gic_dist->config[15] = 0x55555555;

/* Configure per-processor SGI/PPI target registers */
for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) {
for (int i = 0; i < GUEST_NUM_VCPUS; i++) {
for (int j = 0; j < ARRAY_SIZE(gic_dist->targets0[i]); j++) {
for (int irq = 0; irq < sizeof(uint32_t); irq++) {
gic_dist->targets0[i][j] |= ((1 << i) << (irq * 8));
Expand All @@ -109,23 +109,24 @@ static void vgic_dist_reset(struct gic_dist_map *gic_dist)
void vgic_init()
{
memset(&vgic, 0, sizeof(vgic_t));
for (int i = 0; i < NUM_SLOTS_SPI_VIRQ; i++) {
vgic.vspis[i].virq = VIRQ_INVALID;
}
for (int i = 0; i < NUM_VCPU_LOCAL_VIRQS; i++) {
vgic.vgic_vcpu[GUEST_VCPU_ID].local_virqs[i].virq = VIRQ_INVALID;
}
for (int i = 0; i < NUM_LIST_REGS; i++) {
vgic.vgic_vcpu[GUEST_VCPU_ID].lr_shadow[i].virq = VIRQ_INVALID;
}
for (int i = 0; i < MAX_IRQ_QUEUE_LEN; i++) {
vgic.vgic_vcpu[GUEST_VCPU_ID].irq_queue.irqs[i] = NULL;
}
for (int i = 0; i < NUM_SLOTS_SPI_VIRQ; i++) {
vgic.vspis[i].virq = VIRQ_INVALID;
vgic.vspis[i].ack_fn = NULL;
vgic.vspis[i].ack_data = NULL;

for (int vcpu = 0; vcpu < GUEST_NUM_VCPUS; vcpu++) {
for (int i = 0; i < NUM_VCPU_LOCAL_VIRQS; i++) {
vgic.vgic_vcpu[vcpu].local_virqs[i].virq = VIRQ_INVALID;
}
for (int i = 0; i < NUM_LIST_REGS; i++) {
vgic.vgic_vcpu[vcpu].lr_shadow[i].virq = VIRQ_INVALID;
}
for (int i = 0; i < MAX_IRQ_QUEUE_LEN; i++) {
vgic.vgic_vcpu[vcpu].irq_queue.irqs[i] = NULL;
}
for (int i = 0; i < NUM_SLOTS_SPI_VIRQ; i++) {
vgic.vspis[i].virq = VIRQ_INVALID;
vgic.vspis[i].ack_fn = NULL;
vgic.vspis[i].ack_data = NULL;
}
}

vgic.registers = &dist;
memset(vgic.registers, 0, sizeof(struct gic_dist_map));
vgic_dist_reset(vgic_get_dist(vgic.registers));
Expand Down
1 change: 1 addition & 0 deletions src/arch/aarch64/vgic/vgic_v3.c
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ vgic_reg_t vgic_regs;
void vgic_init()
{
// @ivanv: audit
// TODO: fix for SMP
for (int i = 0; i < NUM_SLOTS_SPI_VIRQ; i++) {
vgic.vspis[i].virq = VIRQ_INVALID;
}
Expand Down
30 changes: 16 additions & 14 deletions src/arch/aarch64/virq.c
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,22 @@ bool virq_controller_init(size_t boot_vcpu_id) {
}
#endif

success = vgic_register_irq(boot_vcpu_id, PPI_VTIMER_IRQ, &vppi_event_ack, NULL);
if (!success) {
LOG_VMM_ERR("Failed to register vCPU virtual timer IRQ: 0x%lx\n", PPI_VTIMER_IRQ);
return false;
}
success = vgic_register_irq(boot_vcpu_id, SGI_RESCHEDULE_IRQ, &sgi_ack, NULL);
if (!success) {
LOG_VMM_ERR("Failed to register vCPU SGI 0 IRQ");
return false;
}
success = vgic_register_irq(boot_vcpu_id, SGI_FUNC_CALL, &sgi_ack, NULL);
if (!success) {
LOG_VMM_ERR("Failed to register vCPU SGI 1 IRQ");
return false;
for (int vcpu = 0; vcpu < GUEST_NUM_VCPUS; vcpu++) {
success = vgic_register_irq(vcpu, PPI_VTIMER_IRQ, &vppi_event_ack, NULL);
if (!success) {
LOG_VMM_ERR("Failed to register vCPU virtual timer IRQ: 0x%lx\n", PPI_VTIMER_IRQ);
return false;
}
success = vgic_register_irq(vcpu, SGI_RESCHEDULE_IRQ, &sgi_ack, NULL);
if (!success) {
LOG_VMM_ERR("Failed to register vCPU SGI 0 IRQ");
return false;
}
success = vgic_register_irq(vcpu, SGI_FUNC_CALL, &sgi_ack, NULL);
if (!success) {
LOG_VMM_ERR("Failed to register vCPU SGI 1 IRQ");
return false;
}
}

return true;
Expand Down
2 changes: 2 additions & 0 deletions src/guest.c
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ bool guest_start(size_t boot_vcpu_id, uintptr_t kernel_pc, uintptr_t dtb, uintpt
}
LOG_VMM("starting guest at 0x%lx, DTB at 0x%lx, initial RAM disk at 0x%lx\n",
regs.pc, regs.x0, initrd);

vcpu_set_on(boot_vcpu_id, true);
/* Restart the boot vCPU to the program counter of the TCB associated with it */
microkit_vcpu_restart(boot_vcpu_id, regs.pc);

Expand Down

0 comments on commit 7e4f2b9

Please sign in to comment.