Change the way stage 2 page table is managed

lib/armv9a/src/macro.rs

@@ -137,6 +137,11 @@ macro_rules! define_bits {
                 $name(data)
             }
 
+            #[inline(always)]
+            pub fn get_mut(&mut self) -> &mut Self {
+                self
+            }
+
             #[inline(always)]
             pub fn get(&self) -> u64 {
                 self.0

lib/armv9a/src/regs.rs

@@ -50,7 +50,17 @@ define_bits!(
     DFSC[5 - 0]
 );
 
+pub const ESR_EL2_EC_UNKNOWN: u64 = 0;
+pub const ESR_EL2_EC_WFX: u64 = 1;
+pub const ESR_EL2_EC_FPU: u64 = 7;
+pub const ESR_EL2_EC_SVC: u64 = 21;
+pub const ESR_EL2_EC_HVC: u64 = 22;
+pub const ESR_EL2_EC_SMC: u64 = 23;
+pub const ESR_EL2_EC_SYSREG: u64 = 24;
+pub const ESR_EL2_EC_SVE: u64 = 25;
+pub const ESR_EL2_EC_INST_ABORT: u64 = 32;
 pub const ESR_EL2_EC_DATA_ABORT: u64 = 36;
+pub const ESR_EL2_EC_SERROR: u64 = 47;
 
 define_register!(SP);
 define_sys_register!(SP_EL0);
@@ -135,6 +145,30 @@ define_sys_register!(
     PARange[03 - 00]
 );
 
+define_sys_register!(
+    ID_AA64MMFR1_EL1, // ref. D19.2.65: AArch64 Memory Model Feature Register 1
+    CMOW[59 - 56],
+    TIDCP1[55 - 52],
+    nTLBPA[51 - 48],
+    AFP[47 - 44],
+    HCX[43 - 40],
+    ETS[39 - 36],
+    TWED[35 - 32],
+    XNX[31 - 28],
+    SpecSEI[27 - 24],
+    PAN[23 - 20],
+    LO[19 - 16],
+    HPDS[15 - 12],
+    VH[11 - 8],
+    VMID[7 - 4],
+    HAFDBS[3 - 0]
+);
+
+pub mod mmfr1_vmid {
+    pub const VMIDBITS_8: u64 = 0;
+    pub const VMIDBITS_16: u64 = 2;
+}
+
 define_sys_register!(
     MAIR_EL2, // ref. D7.2.71: Memory Attribute Indirection Register
     Attr7[63 - 56],
@@ -183,7 +217,26 @@ define_sys_register!(
 );
 
 define_sys_register!(
-    VTCR_EL2,
+    VTCR_EL2, // ref. Virtualzation Translation Control Register
+    DS[32 - 32],
+    RES1[31 - 31],
+    // Non-secure stage 2 translation output address space for the Secure EL1&0
+    // translation regime
+    // 0b0: All stage 2 translations for the Non-secure IPA space of the Secure EL1&0
+    //      translation regime acccess the Secure PA space
+    // 0b1: All stage 2 translations for the Non-secure IPA space of the secure EL1&0
+    //      trnaslation regmime access the non-secure PA space
+    NSA[30 - 30],
+    // Non-secure stage 2 translation table address space for the Secure EL1&0
+    // translation regime
+    NSW[29 - 29],
+    HWU62[28 - 28],
+    HWU61[27 - 27],
+    HWU60[26 - 26],
+    HWU59[25 - 25],
+    RES0[24 - 23],
+    HD[22 - 22],
+    HA[21 - 21],
     VS[19 - 19],    // VMID size. 0b0: 8bits, 0b1: 16bit
     PS[18 - 16],    // Physical address size for the second stage of translation
     TG0[15 - 14],   // Granule size (VTTBR_EL2)
@@ -194,6 +247,13 @@ define_sys_register!(
     T0SZ[5 - 0]     // Size offset of the memory region (TTBR0_EL2)
 );
 
+pub mod vtcr_sl0 {
+    pub const SL0_4K_L2: u64 = 0x0;
+    pub const SL0_4K_L1: u64 = 0x1;
+    pub const SL0_4K_L0: u64 = 0x2;
+    pub const SL0_4K_L3: u64 = 0x3;
+}
+
 pub mod tcr_paddr_size {
     // PS
     pub const PS_4G: u64 = 0b000; // 32bits
@@ -246,6 +306,12 @@ define_sys_register!(
     FIPA[43 - 4] //
 );
 
+define_bits!(
+    HpfarEl2, // Ref. D13.2.55
+    NS[63 - 63],
+    FIPA[43 - 4] //
+);
+
 define_sys_register!(
     FAR_EL2, // Ref. D13.2.55
     OFFSET[11 - 0]
@@ -386,7 +452,6 @@ define_iss_id!(ISS_ID_AA64ISAR1_EL1, 3, 0, 0, 6, 1);
 
 define_iss_id!(ISS_ID_AA64MMFR0_EL1, 3, 0, 0, 7, 0);
 
-define_sys_register!(ID_AA64MMFR1_EL1);
 define_iss_id!(ISS_ID_AA64MMFR1_EL1, 3, 0, 0, 7, 1);
 
 define_sys_register!(ID_AA64MMFR2_EL1);

lib/vmsa/src/page_table.rs

@@ -31,6 +31,7 @@ pub trait Entry {
     fn clear(&mut self);
 
     fn pte(&self) -> u64;
+    fn mut_pte(&mut self) -> &mut Self::Inner;
     fn address(&self, level: usize) -> Option<PhysAddr>;
 
     fn set(&mut self, addr: PhysAddr, flags: u64, is_raw: bool) -> Result<(), Error>;
@@ -82,6 +83,7 @@ pub struct PageTable<A, L, E, const N: usize> {
 }
 
 pub trait PageTableMethods<A: Address, L, E: Entry, const N: usize> {
+    fn new_with_base(base: usize) -> Result<*mut PageTable<A, L, E, N>, Error>;
     fn new_with_align(size: usize, align: usize) -> Result<*mut PageTable<A, L, E, N>, Error>;
     /// Sets multiple page table entries
     ///
@@ -170,6 +172,15 @@ impl<A: Address, L: Level, E: Entry, const N: usize> PageTableMethods<A, L, E, N
         Ok(table)
     }
 
+    fn new_with_base(base: usize) -> Result<*mut PageTable<A, L, E, N>, Error> {
+        let table = base as *mut PageTable<A, L, E, N>;
+        unsafe {
+            let arr: [E; N] = core::array::from_fn(|_| E::new());
+            (*table).entries = arr;
+        }
+        Ok(table)
+    }
+
     fn set_pages<S: PageSize>(
         &mut self,
         guest: PageIter<S, A>,
@@ -278,20 +289,33 @@ where
         }
         let index = E::index::<L>(page.address().into());
 
-        match self.entries[index].is_valid() {
-            true => {
-                if L::THIS_LEVEL < level {
-                    // Need to go deeper (recursive)
-                    match self.subtable::<S>(page) {
-                        Ok(subtable) => subtable.entry(page, level, no_valid_check, func),
-                        Err(e) => Err(e),
+        if no_valid_check {
+            if L::THIS_LEVEL < level {
+                // Need to go deeper (recursive)
+                match self.subtable::<S>(page) {
+                    Ok(subtable) => subtable.entry(page, level, no_valid_check, func),
+                    Err(_e) => Ok((None, L::THIS_LEVEL)),
+                }
+            } else {
+                // The page is either LargePage or HugePage
+                Ok((func(&mut self.entries[index])?, L::THIS_LEVEL))
+            }
+        } else {
+            match self.entries[index].is_valid() {
+                true => {
+                    if L::THIS_LEVEL < level {
+                        // Need to go deeper (recursive)
+                        match self.subtable::<S>(page) {
+                            Ok(subtable) => subtable.entry(page, level, no_valid_check, func),
+                            Err(_e) => Ok((None, L::THIS_LEVEL)),
+                        }
+                    } else {
+                        // The page is either LargePage or HugePage
+                        Ok((func(&mut self.entries[index])?, L::THIS_LEVEL))
                     }
-                } else {
-                    // The page is either LargePage or HugePage
-                    Ok((func(&mut self.entries[index])?, L::THIS_LEVEL))
                 }
+                false => Err(Error::MmNoEntry),
             }
-            false => Err(Error::MmNoEntry),
         }
     }
 

plat/fvp/Cargo.toml

@@ -16,6 +16,7 @@ max_level_info  = ["log/max_level_info", "islet_rmm/max_level_info"]
 max_level_debug = ["log/max_level_debug", "islet_rmm/max_level_debug"]
 max_level_trace = ["log/max_level_trace", "islet_rmm/max_level_trace"]
 stat = ["islet_rmm/stat"]
+realm_linux = ["islet_rmm/realm_linux"]
 
 [dependencies]
 armv9a = { path = "../../lib/armv9a" }

rmm/Cargo.toml

@@ -25,4 +25,5 @@ max_level_warn  = ["log/max_level_warn"]
 max_level_info  = ["log/max_level_info"]
 max_level_debug = ["log/max_level_debug"]
 max_level_trace = ["log/max_level_trace"]
+realm_linux = []
 stat = []

rmm/src/exception/trap.rs

@@ -2,10 +2,12 @@ mod frame;
 pub mod syndrome;
 
 use self::frame::TrapFrame;
+use self::syndrome::Fault;
 use self::syndrome::Syndrome;
 use super::lower::synchronous;
 use crate::cpu;
 use crate::event::realmexit;
+use crate::mm::translation::PageTable;
 use crate::realm::context::Context;
 use crate::realm::vcpu::VCPU;
 
@@ -52,8 +54,58 @@ pub extern "C" fn handle_exception(info: Info, esr: u32, tf: &mut TrapFrame) {
                 debug!("{:?}\nESR: {:X}\n{:#X?}", info, esr, tf);
                 tf.elr += 4; //continue
             }
+            Syndrome::PCAlignmentFault => {
+                debug!("PCAlignmentFault");
+            }
+            Syndrome::DataAbort(fault) => {
+                let far = unsafe { FAR_EL2.get() };
+                debug!("Data Abort (higher), far:{:X}", far);
+                match fault {
+                    Fault::AddressSize { level } => {
+                        debug!("address size, level:{}", level);
+                    }
+                    Fault::Translation { level } => {
+                        debug!("translation, level:{}, esr:{:X}", level, esr);
+                        PageTable::get_ref().map(far as usize, true);
+                        tf.elr += 4; //continue
+                    }
+                    Fault::AccessFlag { level } => {
+                        debug!("access flag, level:{}", level);
+                    }
+                    Fault::Permission { level } => {
+                        debug!("permission, level:{}", level);
+                    }
+                    Fault::Alignment => {
+                        debug!("alignment");
+                    }
+                    Fault::TLBConflict => {
+                        debug!("tlb conflict");
+                    }
+                    Fault::Other(_x) => {
+                        debug!("other");
+                    }
+                }
+            }
+            Syndrome::InstructionAbort(v) => {
+                debug!("Instruction Abort (higher)");
+            }
+            Syndrome::HVC => {
+                debug!("HVC");
+            }
+            Syndrome::SMC => {
+                debug!("SMC");
+            }
+            Syndrome::SysRegInst => {
+                debug!("SysRegInst");
+            }
+            Syndrome::WFX => {
+                debug!("WFX");
+            }
+            Syndrome::Other(v) => {
+                debug!("Other");
+            }
             undefined => {
-                panic!("{:?} and {:?} on CPU {:?}", info, esr, cpu::id());
+                panic!("{:?} and ESR: {:X} on CPU {:?}", info, esr, cpu::id());
             }
         },
         _ => {

rmm/src/exception/trap/syndrome.rs

@@ -52,9 +52,23 @@ impl From<u32> for Syndrome {
             0b01_0011 => Syndrome::SMC,
             0b01_0111 => Syndrome::SMC,
             0b01_1000 => Syndrome::SysRegInst,
-            0b10_0000 | 0b10_0001 => Syndrome::InstructionAbort(Fault::from(origin)),
+            0b10_0000 => {
+                debug!("Instruction Abort from a lower Exception level");
+                Syndrome::InstructionAbort(Fault::from(origin))
+            }
+            0b10_0001 => {
+                debug!("Instruction Abort taken without a change in Exception level");
+                Syndrome::InstructionAbort(Fault::from(origin))
+            }
             0b10_0010 => Syndrome::PCAlignmentFault,
-            0b10_0100 | 0b10_0101 => Syndrome::DataAbort(Fault::from(origin)),
+            0b10_0100 => {
+                debug!("Data Abort from a lower Exception level");
+                Syndrome::DataAbort(Fault::from(origin))
+            }
+            0b10_0101 => {
+                debug!("Data Abort without a change in Exception level");
+                Syndrome::DataAbort(Fault::from(origin))
+            }
             0b10_0110 => Syndrome::SPAlignmentFault,
             0b11_1100 => Syndrome::Brk((origin & ESR_EL2::ISS_BRK_CMT as u32) as u16),
             ec => Syndrome::Other(ec as u32),

rmm/src/granule/entry.rs

@@ -231,6 +231,10 @@ impl page_table::Entry for Entry {
         todo!();
     }
 
+    fn mut_pte(&mut self) -> &mut Self::Inner {
+        self.0.get_mut()
+    }
+
     fn address(&self, _level: usize) -> Option<PhysAddr> {
         Some(PhysAddr::from(self.0.lock().addr()))
     }

rmm/src/lib.rs

@@ -52,7 +52,6 @@ use armv9a::{bits_in_reg, regs::*};
 pub unsafe fn start() {
     setup_mmu_cfg();
     setup_el2();
-    activate_stage2_mmu();
     setup_gst();
 
     Monitor::new().run();
@@ -84,25 +83,6 @@ unsafe fn setup_el2() {
     ICC_SRE_EL2.set(ICC_SRE_EL2::ENABLE | ICC_SRE_EL2::DIB | ICC_SRE_EL2::DFB | ICC_SRE_EL2::SRE);
 }
 
-unsafe fn activate_stage2_mmu() {
-    // stage 2 intitial table: L1 with 1024 entries (2 continuous 4KB pages)
-    let vtcr_el2: u64 = bits_in_reg(VTCR_EL2::PS, tcr_paddr_size::PS_1T)
-        | bits_in_reg(VTCR_EL2::TG0, tcr_granule::G_4K)
-        | bits_in_reg(VTCR_EL2::SH0, tcr_shareable::INNER)
-        | bits_in_reg(VTCR_EL2::ORGN0, tcr_cacheable::WBWA)
-        | bits_in_reg(VTCR_EL2::IRGN0, tcr_cacheable::WBWA)
-        | bits_in_reg(VTCR_EL2::SL0, tcr_start_level::L1)
-        | bits_in_reg(VTCR_EL2::T0SZ, 24); // T0SZ, input address is 2^40 bytes
-
-    // Invalidate the local I-cache so that any instructions fetched
-    // speculatively are discarded.
-    core::arch::asm!("ic iallu", "dsb nsh", "isb",);
-
-    VTCR_EL2.set(vtcr_el2);
-
-    core::arch::asm!("tlbi alle2", "dsb ish", "isb",);
-}
-
 unsafe fn setup_mmu_cfg() {
     core::arch::asm!("tlbi alle2is", "dsb ish", "isb",);
 

rmm/src/mm/page_table/entry.rs

@@ -46,6 +46,10 @@ impl page_table::Entry for Entry {
         self.0.get()
     }
 
+    fn mut_pte(&mut self) -> &mut Self::Inner {
+        self.0.get_mut()
+    }
+
     fn address(&self, level: usize) -> Option<PhysAddr> {
         match self.is_valid() {
             true => match self.0.get_masked_value(PTDesc::TYPE) {

rmm/src/realm/mm/mod.rs

@@ -13,17 +13,10 @@ use address::{GuestPhysAddr, PhysAddr};
 
 pub trait IPATranslation: Debug + Send + Sync {
     fn get_base_address(&self) -> *const c_void;
-    fn set_pages(
-        &mut self,
-        guest: GuestPhysAddr,
-        phys: PhysAddr,
-        size: usize,
-        flags: usize,
-        is_raw: bool,
-    ) -> Result<(), Error>;
-    fn unset_pages(&mut self, guest: GuestPhysAddr, size: usize);
     // TODO: remove mut
     fn ipa_to_pa(&mut self, guest: GuestPhysAddr, level: usize) -> Option<PhysAddr>;
     fn ipa_to_pte(&mut self, guest: GuestPhysAddr, level: usize) -> Option<(u64, usize)>;
+    fn ipa_to_pte_set(&mut self, guest: GuestPhysAddr, level: usize, val: u64)
+        -> Result<(), Error>;
     fn clean(&mut self);
 }