From 31fff9059a96d979f641561617d40e9820b94d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Wed, 8 May 2024 22:10:11 +0200 Subject: [PATCH] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/x86_64/kernel/acpi.rs | 146 +++++++++++++- src/arch/x86_64/kernel/apic.rs | 87 ++++++--- src/arch/x86_64/kernel/mod.rs | 22 +++ src/arch/x86_64/kernel/scheduler.rs | 57 +++--- src/arch/x86_64/mm/paging.rs | 291 +++++++++++++++++++++++++--- src/arch/x86_64/mm/physicalmem.rs | 71 ++++++- src/arch/x86_64/mm/virtualmem.rs | 23 ++- src/drivers/virtio/env.rs | 22 ++- src/mm/mod.rs | 47 +++-- 9 files changed, 650 insertions(+), 116 deletions(-) diff --git a/src/arch/x86_64/kernel/acpi.rs b/src/arch/x86_64/kernel/acpi.rs index 104b578f88..550aff5401 100644 --- a/src/arch/x86_64/kernel/acpi.rs +++ b/src/arch/x86_64/kernel/acpi.rs @@ -85,6 +85,9 @@ impl AcpiSdtHeader { fn signature(&self) -> &str { str::from_utf8(&self.signature).unwrap() } + pub fn header_start_address(&self) -> usize { + self as *const _ as usize + } } /// A convenience structure to work with an ACPI table. @@ -308,6 +311,16 @@ fn detect_rsdp(start_address: PhysAddr, end_address: PhysAddr) -> Result<&'stati /// Detects ACPI support of the computer system. /// Returns a reference to the ACPI RSDP within the Ok() if successful or an empty Err() on failure. fn detect_acpi() -> Result<&'static AcpiRsdp, ()> { + // For UEFI Systems, RSDP detection takes place in the bootloader and the tables are already mapped so we only need to return a proper reference to the table + if crate::arch::kernel::is_uefi() { + let rsdp = crate::arch::kernel::get_rsdp_addr(); + trace!("RSDP detected successfully at {rsdp:#x?}"); + let rsdp = unsafe { &*(rsdp as *const AcpiRsdp) }; + if &rsdp.signature != b"RSD PTR " { + panic!("RSDP Address not valid!"); + } + return Ok(rsdp); + } // Get the address of the EBDA. let frame = PhysFrame::::containing_address(x86_64::PhysAddr::new(EBDA_PTR_LOCATION.0)); @@ -412,7 +425,26 @@ fn parse_fadt(fadt: AcpiTable<'_>) { } else { PhysAddr(fadt_table.dsdt.into()) }; - let dsdt = AcpiTable::map(dsdt_address); + let dsdt = if !crate::arch::kernel::is_uefi() { + AcpiTable::map(dsdt_address) + } else { + // For UEFI Systems, the tables are already mapped so we only need to return a proper reference to the table + let table = unsafe { (dsdt_address.0 as *const AcpiSdtHeader).as_ref().unwrap() }; + let mut length = table.length as u64; + let res = length % BasePageSize::SIZE; + //ACPI tables are 4KiB aligned, so the length can span the entire pagetable + //-> necessary since the tables are dropped (and virtual memory is deallocated) after use + //somehow, this table is larger than 4KiB, so we bumb it up to the entire page + if res != 0 { + length += 0x1000 - res; + } + + AcpiTable { + header: table, + allocated_virtual_address: VirtAddr(dsdt_address.0), + allocated_length: length as usize, + } + }; // Check it. assert!( @@ -462,6 +494,11 @@ pub fn poweroff() { } pub fn init() { + if super::is_uefi() { + init_uefi(); + return; + } + // Detect the RSDP and get a pointer to either the XSDT (64-bit) or RSDT (32-bit), whichever is available. // Both are called RSDT in the following. let rsdp = detect_acpi().expect("Hermit requires an ACPI-compliant system"); @@ -525,3 +562,110 @@ pub fn init() { } } } + +pub fn init_uefi() { + // Retrieve the RSDP and get a pointer to either the XSDT (64-bit) or RSDT (32-bit), whichever is available. + // Both are called RSDT in the following. + let rsdp = detect_acpi().expect("Hermit requires an ACPI-compliant system"); + let rsdt_physical_address = if rsdp.revision >= 2 { + PhysAddr(rsdp.xsdt_physical_address) + } else { + PhysAddr(rsdp.rsdt_physical_address.into()) + }; + + //Load RSDT + let rsdt = &unsafe { *(rsdt_physical_address.0 as *const AcpiSdtHeader) }; + + // The RSDT contains pointers to all available ACPI tables. + // Iterate through them. + let mut current_address = rsdt_physical_address.0 as usize + mem::size_of::(); + trace!("RSDT start address at {current_address:#x}"); + let nr_entries = + (rsdt.length as usize - mem::size_of::()) / mem::size_of::(); + let end_addr = current_address + nr_entries * mem::size_of::(); + while current_address < end_addr { + trace!("current_address: {current_address:#x}"); + + // Depending on the RSDP revision, either an XSDT or an RSDT has been chosen above. + // The XSDT contains 64-bit pointers whereas the RSDT has 32-bit pointers. + let table_physical_address = if rsdp.revision >= 2 { + let address = PhysAddr(unsafe { ptr::read_unaligned(current_address as *const u64) }); + current_address += mem::size_of::(); + address + } else { + let address = + PhysAddr((unsafe { ptr::read_unaligned(current_address as *const u32) }).into()); + current_address += mem::size_of::(); + address + }; + + let table = unsafe { + (table_physical_address.0 as *const AcpiSdtHeader) + .as_ref() + .unwrap() + }; + + let signature = table.signature(); + + debug!("Found ACPI table: {signature:#?}"); + + if signature == "APIC" { + // This is a "Multiple APIC Descriptor Table" (MADT) aka "APIC Table" + // Check and save the entire APIC table for the get_apic_table() call + assert!( + verify_checksum(table.header_start_address(), table.length as usize).is_ok(), + "MADT at {table_physical_address:#x} has invalid checksum" + ); + //ACPI tables are 4KiB aligned, so the length can span the entire pagetable + //-> necessary since the tables are dropped (and virtual memory is deallocated) after use + let mut length = table.length as u64; + if length < 0x1000 { + length = 0x1000; + } + + let madt: AcpiTable<'static> = AcpiTable { + header: table, + allocated_virtual_address: VirtAddr(table_physical_address.0), + allocated_length: length as usize, + }; + MADT.set(madt).unwrap(); + } else if signature == "FACP" { + // This is the "Fixed ACPI Description Table" (FADT) aka "Fixed ACPI Control Pointer" (FACP) + // Check and parse this table for the poweroff() call + assert!( + verify_checksum(table.header_start_address(), table.length as usize).is_ok(), + "FADT at {table_physical_address:#x} has invalid checksum" + ); + //ACPI tables are 4KiB aligned, so the length can span the entire pagetable + //-> necessary since the tables are dropped (and virtual memory is deallocated) after use + let mut length = table.length as u64; + if length < 0x1000 { + length = 0x1000; + } + let fadt: AcpiTable<'static> = AcpiTable { + header: table, + allocated_virtual_address: VirtAddr(table_physical_address.0), + allocated_length: length as usize, + }; + + parse_fadt(fadt); + } else if signature == "SSDT" { + assert!( + verify_checksum(table.header_start_address(), table.length as usize).is_ok(), + "SSDT at {table_physical_address:p} has invalid checksum" + ); + + let mut length = table.length as u64; + if length < 0x1000 { + length = 0x1000; + } + + let ssdt: AcpiTable<'static> = AcpiTable { + header: table, + allocated_virtual_address: VirtAddr(table_physical_address.0), + allocated_length: length as usize, + }; + parse_ssdt(ssdt); + } + } +} diff --git a/src/arch/x86_64/kernel/apic.rs b/src/arch/x86_64/kernel/apic.rs index 53868b1248..655e91f8a5 100644 --- a/src/arch/x86_64/kernel/apic.rs +++ b/src/arch/x86_64/kernel/apic.rs @@ -259,13 +259,18 @@ pub fn local_apic_id_count() -> u32 { } fn init_ioapic_address(phys_addr: PhysAddr) { - let ioapic_address = virtualmem::allocate(BasePageSize::SIZE as usize).unwrap(); - IOAPIC_ADDRESS.set(ioapic_address).unwrap(); - debug!("Mapping IOAPIC at {phys_addr:p} to virtual address {ioapic_address:p}",); + if !crate::kernel::is_uefi() { + let ioapic_address = virtualmem::allocate(BasePageSize::SIZE as usize).unwrap(); + IOAPIC_ADDRESS.set(ioapic_address).unwrap(); + debug!("Mapping IOAPIC at {phys_addr:p} to virtual address {ioapic_address:p}",); - let mut flags = PageTableEntryFlags::empty(); - flags.device().writable().execute_disable(); - paging::map::(ioapic_address, phys_addr, 1, flags); + let mut flags = PageTableEntryFlags::empty(); + flags.device().writable().execute_disable(); + paging::map::(ioapic_address, phys_addr, 1, flags); + } else { + // UEFI systems have already id mapped everything, so we can just set the physical address as the virtual one + IOAPIC_ADDRESS.set(VirtAddr(phys_addr.as_u64())).unwrap(); + } } #[cfg(not(feature = "acpi"))] @@ -469,16 +474,23 @@ pub fn init() { if !processor::supports_x2apic() { // We use the traditional xAPIC mode available on all x86-64 CPUs. // It uses a mapped page for communication. - let local_apic_address = virtualmem::allocate(BasePageSize::SIZE as usize).unwrap(); - LOCAL_APIC_ADDRESS.set(local_apic_address).unwrap(); - debug!( - "Mapping Local APIC at {:p} to virtual address {:p}", - local_apic_physical_address, local_apic_address - ); + if crate::kernel::is_uefi() { + //already id mapped in UEFI systems, just use the physical address as virtual one + LOCAL_APIC_ADDRESS + .set(VirtAddr(local_apic_physical_address.as_u64())) + .unwrap(); + } else { + let local_apic_address = virtualmem::allocate(BasePageSize::SIZE as usize).unwrap(); + LOCAL_APIC_ADDRESS.set(local_apic_address).unwrap(); + debug!( + "Mapping Local APIC at {:p} to virtual address {:p}", + local_apic_physical_address, local_apic_address + ); - let mut flags = PageTableEntryFlags::empty(); - flags.device().writable().execute_disable(); - paging::map::(local_apic_address, local_apic_physical_address, 1, flags); + let mut flags = PageTableEntryFlags::empty(); + flags.device().writable().execute_disable(); + paging::map::(local_apic_address, local_apic_physical_address, 1, flags); + } } // Set gates to ISRs for the APIC interrupts we are going to enable. @@ -696,6 +708,8 @@ pub fn init_next_processor_variables() { pub fn boot_application_processors() { use core::hint; + use x86_64::structures::paging::Translate; + use super::{raw_boot_info, start}; let smp_boot_code = include_bytes!(concat!(core::env!("OUT_DIR"), "/boot.bin")); @@ -706,20 +720,37 @@ pub fn boot_application_processors() { "SMP Boot Code is larger than a page" ); debug!("SMP boot code is {} bytes long", smp_boot_code.len()); + // We can only allocate full pages of physmem - // Identity-map the boot code page and copy over the code. - debug!( - "Mapping SMP boot code to physical and virtual address {:p}", - SMP_BOOT_CODE_ADDRESS - ); - let mut flags = PageTableEntryFlags::empty(); - flags.normal().writable(); - paging::map::( - SMP_BOOT_CODE_ADDRESS, - PhysAddr(SMP_BOOT_CODE_ADDRESS.as_u64()), - 1, - flags, - ); + if crate::kernel::is_uefi() { + // Since UEFI already provides identity-mapped pagetables, we only have to check whether the physical address 0x8000 really is mapped to the virtual address 0x8000 + unsafe { + let pt = crate::arch::mm::paging::identity_mapped_page_table(); + let virt_addr = 0x8000; + let phys_addr = pt + .translate_addr(x86_64::VirtAddr::new(virt_addr)) + .unwrap() + .as_u64(); + assert_eq!( + phys_addr, virt_addr, + "0x8000 is not identity-mapped for SMP boot code!" + ) + } + } else { + // Identity-map the boot code page and copy over the code. + debug!( + "Mapping SMP boot code to physical and virtual address {:p}", + SMP_BOOT_CODE_ADDRESS + ); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable(); + paging::map::( + SMP_BOOT_CODE_ADDRESS, + PhysAddr(SMP_BOOT_CODE_ADDRESS.as_u64()), + 1, + flags, + ); + } unsafe { ptr::copy_nonoverlapping( smp_boot_code.as_ptr(), diff --git a/src/arch/x86_64/kernel/mod.rs b/src/arch/x86_64/kernel/mod.rs index aef2bc5768..14afb13d49 100644 --- a/src/arch/x86_64/kernel/mod.rs +++ b/src/arch/x86_64/kernel/mod.rs @@ -67,10 +67,32 @@ pub fn get_image_size() -> usize { (range.end - range.start) as usize } +pub fn get_start() -> usize { + boot_info().hardware_info.phys_addr_range.start as usize +} + pub fn get_limit() -> usize { boot_info().hardware_info.phys_addr_range.end as usize } +pub fn is_uefi() -> bool { + let fdt_addr = get_fdt().unwrap(); + let fdt = unsafe { fdt::Fdt::from_ptr(fdt_addr as *const u8).unwrap() }; + fdt.root().compatible().first() == "hermit,uefi" +} + +pub fn get_rsdp_addr() -> u64 { + let fdt_addr = get_fdt().unwrap(); + let fdt = unsafe { fdt::Fdt::from_ptr(fdt_addr as *const u8).unwrap() }; + fdt.find_node("/hermit,rsdp") + .unwrap() + .reg() + .unwrap() + .next() + .unwrap() + .starting_address as u64 +} + pub fn get_mbinfo() -> Option { match boot_info().platform_info { PlatformInfo::Multiboot { diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index c81ae9a57f..ebb6e1dde1 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -116,29 +116,30 @@ impl TaskStacks { let mut flags = PageTableEntryFlags::empty(); flags.normal().writable().execute_disable(); - // map IST1 into the address space - crate::arch::mm::paging::map::( - virt_addr + BasePageSize::SIZE, - phys_addr, - IST_SIZE / BasePageSize::SIZE as usize, - flags, - ); - - // map kernel stack into the address space - crate::arch::mm::paging::map::( - virt_addr + IST_SIZE + 2 * BasePageSize::SIZE, - phys_addr + IST_SIZE, - DEFAULT_STACK_SIZE / BasePageSize::SIZE as usize, - flags, - ); - - // map user stack into the address space - crate::arch::mm::paging::map::( - virt_addr + IST_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE, - phys_addr + IST_SIZE + DEFAULT_STACK_SIZE, - user_stack_size / BasePageSize::SIZE as usize, - flags, - ); + // For UEFI systems, the entire memory is already mapped, just clear the stack for safety + if !crate::kernel::is_uefi() { + // map IST1 into the address space + crate::arch::mm::paging::map::( + virt_addr + BasePageSize::SIZE, + phys_addr, + IST_SIZE / BasePageSize::SIZE as usize, + flags, + ); + // map kernel stack into the address space + crate::arch::mm::paging::map::( + virt_addr + IST_SIZE + 2 * BasePageSize::SIZE, + phys_addr + IST_SIZE, + DEFAULT_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + // map user stack into the address space + crate::arch::mm::paging::map::( + virt_addr + IST_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE, + phys_addr + IST_SIZE + DEFAULT_STACK_SIZE, + user_stack_size / BasePageSize::SIZE as usize, + flags, + ); + } // clear user stack unsafe { @@ -225,10 +226,12 @@ impl Drop for TaskStacks { stacks.total_size >> 10, ); - crate::arch::mm::paging::unmap::( - stacks.virt_addr, - stacks.total_size / BasePageSize::SIZE as usize + 4, - ); + if !crate::kernel::is_uefi() { + crate::arch::mm::paging::unmap::( + stacks.virt_addr, + stacks.total_size / BasePageSize::SIZE as usize + 4, + ); + } crate::arch::mm::virtualmem::deallocate( stacks.virt_addr, stacks.total_size + 4 * BasePageSize::SIZE as usize, diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 8e80da762b..3ab0b8b048 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -2,15 +2,20 @@ use core::fmt::Debug; use core::ptr; use x86_64::instructions::tlb; -use x86_64::registers::control::Cr2; +use x86_64::registers::control::{Cr0, Cr0Flags, Cr2, Cr3}; #[cfg(feature = "common-os")] use x86_64::registers::segmentation::SegmentSelector; pub use x86_64::structures::idt::InterruptStackFrame as ExceptionStackFrame; use x86_64::structures::idt::PageFaultErrorCode; -use x86_64::structures::paging::mapper::{TranslateResult, UnmapError}; +use x86_64::structures::paging::frame::PhysFrameRange; +use x86_64::structures::paging::mapper::{ + MapToError, MappedFrame, MapperAllSizes, TranslateResult, UnmapError, +}; +use x86_64::structures::paging::page_table::PageTableLevel; pub use x86_64::structures::paging::PageTableFlags as PageTableEntryFlags; use x86_64::structures::paging::{ - Mapper, Page, PageTableIndex, PhysFrame, RecursivePageTable, Size2MiB, Translate, + Mapper, OffsetPageTable, Page, PageTable, PageTableIndex, PhysFrame, RecursivePageTable, + Size1GiB, Size2MiB, Size4KiB, Translate, }; use crate::arch::x86_64::kernel::processor; @@ -89,6 +94,7 @@ pub use x86_64::structures::paging::{ PageSize, Size1GiB as HugePageSize, Size2MiB as LargePageSize, Size4KiB as BasePageSize, }; +/// Returns a recursive page table mapping, its last entry is mapped to the table itself unsafe fn recursive_page_table() -> RecursivePageTable<'static> { let level_4_table_addr = 0xFFFF_FFFF_FFFF_F000; let level_4_table_ptr = ptr::with_exposed_provenance_mut(level_4_table_addr); @@ -98,6 +104,17 @@ unsafe fn recursive_page_table() -> RecursivePageTable<'static> { } } +/// Returns a mapping of the physical memory where physical address is equal to the virtual address (no offset) +pub unsafe fn identity_mapped_page_table() -> OffsetPageTable<'static> { + let level_4_table_addr = Cr3::read().0.start_address().as_u64(); + let level_4_table_ptr = + ptr::with_exposed_provenance_mut::(level_4_table_addr.try_into().unwrap()); + unsafe { + let level_4_table = &mut *(level_4_table_ptr); + OffsetPageTable::new(level_4_table, x86_64::addr::VirtAddr::new(0x0)) + } +} + /// Translate a virtual memory address to a physical one. pub fn virtual_to_physical(virtual_address: VirtAddr) -> Option { let virtual_address = x86_64::VirtAddr::new(virtual_address.0); @@ -137,7 +154,8 @@ pub fn map( flags: PageTableEntryFlags, ) where S: PageSize + Debug, - RecursivePageTable<'static>: Mapper, + RecursivePageTable<'static>: Mapper + MapperAllSizes, + OffsetPageTable<'static>: Mapper + MapperAllSizes, { let pages = { let start = Page::::containing_address(x86_64::VirtAddr::new(virtual_address.0)); @@ -158,25 +176,67 @@ pub fn map( #[cfg(feature = "smp")] let mut ipi_tlb_flush = false; - let mut frame_allocator = physicalmem::PHYSICAL_FREE_LIST.lock(); - for (page, frame) in pages.zip(frames) { - unsafe { - // TODO: Require explicit unmaps - if let Ok((_frame, flush)) = recursive_page_table().unmap(page) { - #[cfg(feature = "smp")] - { - ipi_tlb_flush = true; + if !crate::arch::x86_64::kernel::is_uefi() { + for (page, frame) in pages.zip(frames) { + unsafe { + // TODO: Require explicit unmaps + if let Ok((_frame, flush)) = recursive_page_table().unmap(page) { + #[cfg(feature = "smp")] + { + ipi_tlb_flush = true; + } + flush.flush(); + debug!("Had to unmap page {page:?} before mapping."); } - flush.flush(); - debug!("Had to unmap page {page:?} before mapping."); + recursive_page_table() + .map_to( + page, + frame, + flags, + &mut *physicalmem::PHYSICAL_FREE_LIST.lock(), + ) + .unwrap() + .flush(); } - recursive_page_table() - .map_to(page, frame, flags, &mut *frame_allocator) - .unwrap() + } + } else { + for (page, frame) in pages.zip(frames) { + unsafe { + let mut pt = identity_mapped_page_table(); + if let Ok((_frame, flush)) = pt.unmap(page) { + trace!("unmapped"); + #[cfg(feature = "smp")] + { + ipi_tlb_flush = true; + } + flush.flush(); + debug!("Had to unmap page {page:?} before mapping."); + } + let res = pt.identity_map( + frame, + flags, + &mut *physicalmem::PHYSICAL_FREE_LIST.lock(), + ); + res.unwrap_or_else(|e| match e { + MapToError::ParentEntryHugePage => { + assert!( + S::SIZE == BasePageSize::SIZE, + "Giant Pages are not supported" + ); + recast_huge_page(pt, page, frame, flags) + } + _ => panic!( + "error {e:?} at: {frame:?} \n pages: 4 {:?}, 3 {:?}, 2 {:?}, 1 {:?} \n", + page.p4_index(), + page.p3_index(), + page.page_table_index(PageTableLevel::Two), + page.page_table_index(PageTableLevel::One) + ), + }) .flush(); + } } } - drop(frame_allocator); #[cfg(feature = "smp")] if ipi_tlb_flush { @@ -190,6 +250,7 @@ pub fn map_heap(virt_addr: VirtAddr, count: usize) -> Result<(), usize> where S: PageSize + Debug, RecursivePageTable<'static>: Mapper, + OffsetPageTable<'static>: Mapper, { let flags = { let mut flags = PageTableEntryFlags::empty(); @@ -300,7 +361,11 @@ pub(crate) extern "x86-interrupt" fn page_fault_handler( scheduler::abort(); } -pub fn init() {} +pub fn init() { + if crate::arch::x86_64::kernel::is_uefi() { + check_root_pagetable(); + } +} pub fn init_page_tables() { if env::is_uhyve() { @@ -309,7 +374,7 @@ pub fn init_page_tables() { // Ideally, uhyve would only map as much memory as necessary, but this requires a hermit-entry ABI jump. // See https://github.com/hermit-os/uhyve/issues/426 let kernel_end_addr = x86_64::VirtAddr::new(mm::kernel_end_address().as_u64()); - let start_page = Page::::from_start_address(kernel_end_addr).unwrap(); + let start_page = Page::::from_start_address(kernel_end_addr).unwrap(); let end_page = Page::from_page_table_indices_2mib( start_page.p4_index(), start_page.p3_index(), @@ -327,11 +392,129 @@ pub fn init_page_tables() { } } -#[allow(dead_code)] -unsafe fn disect(pt: PT, virt_addr: x86_64::VirtAddr) { - use x86_64::structures::paging::mapper::{MappedFrame, TranslateResult}; - use x86_64::structures::paging::{Size1GiB, Size4KiB}; +/// Checks the address stored in the CR3 register and if necessary, makes its page writable. +fn check_root_pagetable() { + let level_4_table_addr = Cr3::read().0.start_address().as_u64(); + let virt_lvl_4_addr = x86_64::VirtAddr::new(level_4_table_addr); + let pt = unsafe { identity_mapped_page_table() }; + unsafe { + Cr0::update(|cr0| cr0.remove(Cr0Flags::WRITE_PROTECT)); + } + match pt.translate(virt_lvl_4_addr) { + TranslateResult::Mapped { + frame, + offset: _, + flags, + } => match frame { + MappedFrame::Size1GiB(_) => { + set_pagetable_page_writable(frame, virt_lvl_4_addr, flags, pt); + } + MappedFrame::Size2MiB(_) => { + set_pagetable_page_writable(frame, virt_lvl_4_addr, flags, pt); + } + MappedFrame::Size4KiB(_) => { + set_pagetable_page_writable(frame, virt_lvl_4_addr, flags, pt); + } + }, + TranslateResult::NotMapped => todo!(), + TranslateResult::InvalidFrameAddress(_) => todo!(), + }; + unsafe { + Cr0::update(|cr0| cr0.insert(Cr0Flags::WRITE_PROTECT)); + } +} +/// This function takes the rootpage and depending on its size (1GiB, 2MiB, 4KiB), changes the flags to make it writable and then flushes the TLB. +/// This is useful for memory manipulation. +fn set_pagetable_page_writable( + framesize: MappedFrame, + addr: x86_64::VirtAddr, + flags: PageTableEntryFlags, + mut pt: OffsetPageTable<'_>, +) { + let page: Page = Page::from_start_address(addr).unwrap(); + match framesize { + MappedFrame::Size1GiB(_) => { + let flush = unsafe { + pt.set_flags_p3_entry(page, flags | PageTableEntryFlags::WRITABLE) + .unwrap() + }; + flush.flush_all(); + } + MappedFrame::Size2MiB(_) => { + let flush = unsafe { + pt.set_flags_p2_entry(page, flags | PageTableEntryFlags::WRITABLE) + .unwrap() + }; + flush.flush_all(); + } + MappedFrame::Size4KiB(_) => { + let flush = unsafe { + pt.update_flags(page, flags | PageTableEntryFlags::WRITABLE) + .unwrap() + }; + flush.flush(); + } + } + trace!("Rootpage now writable"); +} + +/// This function remaps one given Huge Page (2 MiB) into 512 Normal Pages (4 KiB) with a given (Offset/ID mapped) Page Table. +/// It takes the Page Table with its mapping, the Hugepage in question, the physical frame it is stored in and its flags as input. +/// The page gets remapped and a new level 2 Pagetable is allocated in this function to connect the (now) 512 level 1 pages. +unsafe fn recast_huge_page( + mut pt: OffsetPageTable<'static>, + page: Page, + frame: PhysFrame, + flags: PageTableEntryFlags, +) -> x86_64::structures::paging::mapper::MapperFlush +where + S: PageSize + Debug, + OffsetPageTable<'static>: Mapper + MapperAllSizes, +{ + //make sure that the allocated physical frame is NOT inside the page that needs remapping + let forbidden_start: PhysFrame = + PhysFrame::from_start_address(frame.start_address()).unwrap(); + let forbidden_end: PhysFrame = PhysFrame::from_start_address( + x86_64::PhysAddr::new(frame.start_address().as_u64() + 0x200000), + ) + .unwrap(); + let forbidden_range: PhysFrameRange = + PhysFrame::range(forbidden_start, forbidden_end); + //Pagetable walk to get the correct data + let pml4 = pt.level_4_table_mut(); + let pdpte_entry = &mut pml4[page.p4_index()]; + let pdpte_ptr: *mut PageTable = x86_64::VirtAddr::new(pdpte_entry.addr().as_u64()).as_mut_ptr(); + let pdpte = unsafe { &mut *pdpte_ptr }; + let pde_entry = &mut pdpte[page.p3_index()]; + let pde_ptr: *mut PageTable = x86_64::VirtAddr::new(pde_entry.addr().as_u64()).as_mut_ptr(); + let pde = unsafe { &mut *pde_ptr }; + let pte_entry = &mut pde[page.page_table_index(PageTableLevel::Two)]; + let pte_entry_start = pte_entry.addr().as_u64(); //start of HUGE PAGE + let pte_frame: PhysFrame = + physicalmem::allocate_outside_of(S::SIZE as usize, S::SIZE as usize, forbidden_range) + .unwrap(); + + let new_flags = PageTableEntryFlags::PRESENT | PageTableEntryFlags::WRITABLE; + let pte_ptr: *mut PageTable = x86_64::VirtAddr::new(pte_entry.addr().as_u64()).as_mut_ptr(); + let pte = unsafe { &mut *pte_ptr }; + //remap everything + for (i, entry) in pte.iter_mut().enumerate() { + let addr = pte_entry_start + (i * 0x1000) as u64; //calculates starting addresses of the normal sized pages + entry.set_addr(x86_64::PhysAddr::new(addr), new_flags) + } + + pte_entry.set_frame(pte_frame, new_flags); + tlb::flush_all(); // flush TLB to ensure all memory is valid and up-to-date + unsafe { + let mut frame_allocator = physicalmem::PHYSICAL_FREE_LIST.lock(); + pt.identity_map(frame, flags, &mut *frame_allocator) + .unwrap() + } +} + +#[allow(dead_code)] +pub unsafe fn disect(pt: PT, virt_addr: x86_64::VirtAddr) { match pt.translate(virt_addr) { TranslateResult::Mapped { frame, @@ -385,7 +568,7 @@ pub(crate) unsafe fn print_page_tables(levels: usize) { .enumerate() .filter(|(_i, entry)| !entry.is_unused()) { - if level != min_level && i >= 1 { + if level <= min_level { break; } let indent = &" "[0..2 * (4 - level)]; @@ -413,3 +596,61 @@ pub(crate) unsafe fn print_page_tables(levels: usize) { print(pt, 4, 5 - levels); } + +/// This debugging function takes up to 4 PageTable Indices (l2 and l1 are optional because of Huge and Giant Pages) and prints the specific entries in the PageTable. +#[allow(dead_code)] +pub unsafe fn print_specific_pagetable(l4: usize, l3: usize, l2: Option, l1: Option) { + let level_4_table_addr = Cr3::read().0.start_address().as_u64(); + let level_4_table_ptr = + ptr::with_exposed_provenance::(level_4_table_addr.try_into().unwrap()); + let pt = unsafe { &*level_4_table_ptr }; + + for (i, entry) in pt.iter().enumerate() { + if i != l4 { + continue; //we haven't found the correct l4 index yet + } + println!("L4 Entry {i}: {entry:?}"); + let phys = entry.frame().unwrap().start_address(); + let virt = x86_64::VirtAddr::new(phys.as_u64()); + let pt_l3: &PageTable = unsafe { &*virt.as_mut_ptr() }; + for (i, entry) in pt_l3.iter().enumerate() { + if i != l3 { + continue; //we haven't found the correct l3 index yet + } + println!("L3 Entry {i}: {entry:?}"); + + if l2.is_some() { + let phys = entry.frame().unwrap().start_address(); + let virt = x86_64::VirtAddr::new(phys.as_u64()); + let pt_l2: &PageTable = unsafe { &*virt.as_mut_ptr() }; + + for (i, entry) in pt_l2.iter().enumerate() { + if i != l2.unwrap() { + continue; //we haven't found the correct l2 index yet + } + println!("L2 Entry {i}: {entry:?}"); + + if l1.is_some() { + let phys = entry.frame().unwrap().start_address(); + let virt = x86_64::VirtAddr::new(phys.as_u64()); + let pt_l1: &PageTable = unsafe { &*virt.as_mut_ptr() }; + for (i, entry) in pt_l1.iter().enumerate() { + if i != l1.unwrap() { + continue; //we haven't found the correct l1 index yet + } + + println!("L1 Entry {i}: {entry:?}"); + break; + } + } else { + break; + } + } + } else { + break; + } + break; + } + break; + } +} diff --git a/src/arch/x86_64/mm/physicalmem.rs b/src/arch/x86_64/mm/physicalmem.rs index e0983fb49f..460245e441 100644 --- a/src/arch/x86_64/mm/physicalmem.rs +++ b/src/arch/x86_64/mm/physicalmem.rs @@ -3,8 +3,10 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use free_list::{AllocError, FreeList, PageLayout, PageRange}; use hermit_sync::InterruptTicketMutex; use multiboot::information::{MemoryType, Multiboot}; +use x86_64::structures::paging::frame::PhysFrameRange; +use x86_64::structures::paging::PhysFrame; -use crate::arch::x86_64::kernel::{get_fdt, get_limit, get_mbinfo}; +use crate::arch::x86_64::kernel::{get_fdt, get_limit, get_mbinfo, get_start, is_uefi}; use crate::arch::x86_64::mm::paging::{BasePageSize, PageSize}; use crate::arch::x86_64::mm::{MultibootMemory, PhysAddr, VirtAddr}; use crate::{env, mm}; @@ -144,10 +146,30 @@ fn detect_from_uhyve() -> Result<(), ()> { Ok(()) } +/// Use the free memory provided by the UEFI memory map (rewrite as soon as entire memory map is in kernel!) +/// Right now, all memory in the Physical Address Range of HardwareInfo is guaranteed free memory +fn detect_from_memory_map() -> Result<(), ()> { + if !is_uefi() { + return Err(()); + } + + let start = get_start(); + let limit = get_limit(); + + let range = PageRange::new(start, limit).unwrap(); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range); + } + TOTAL_MEMORY.store(limit - start, Ordering::SeqCst); + + Ok(()) +} + pub fn init() { detect_from_fdt() .or_else(|_e| detect_from_multiboot_info()) .or_else(|_e| detect_from_uhyve()) + .or_else(|_e| detect_from_memory_map()) .unwrap(); } @@ -177,6 +199,45 @@ pub fn allocate(size: usize) -> Result { )) } +pub fn allocate_outside_of( + size: usize, + align: usize, + forbidden_range: PhysFrameRange, +) -> Result { + //general sanity checks + assert!(size > 0); + assert!(align > 0); + assert_eq!( + size % align, + 0, + "Size {size:#X} is not a multiple of the given alignment {align:#X}" + ); + assert_eq!( + align % BasePageSize::SIZE as usize, + 0, + "Alignment {:#X} is not a multiple of {:#X}", + align, + BasePageSize::SIZE + ); + + let layout = PageLayout::from_size_align(size, align).unwrap(); + let forbidden_range = PageRange::new( + forbidden_range.start.start_address().as_u64() as usize, + forbidden_range.end.start_address().as_u64() as usize + 4096, + ) + .unwrap(); + + Ok(PhysFrame::from_start_address(x86_64::addr::PhysAddr::new( + PHYSICAL_FREE_LIST + .lock() + .allocate_outside_of(layout, forbidden_range)? + .start() + .try_into() + .unwrap(), + )) + .unwrap()) +} + pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); assert!(align > 0); @@ -208,10 +269,10 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result= PhysAddr(mm::kernel_end_address().as_u64()), - "Physical address {physical_address:p} is not >= KERNEL_END_ADDRESS" - ); + // assert!( + // physical_address >= PhysAddr(mm::kernel_end_address().as_u64()), + // "Physical address {physical_address:p} is not >= KERNEL_END_ADDRESS" + // ); assert!(size > 0); assert_eq!( size % BasePageSize::SIZE as usize, diff --git a/src/arch/x86_64/mm/virtualmem.rs b/src/arch/x86_64/mm/virtualmem.rs index a3bd38d788..ddf6607617 100644 --- a/src/arch/x86_64/mm/virtualmem.rs +++ b/src/arch/x86_64/mm/virtualmem.rs @@ -1,6 +1,7 @@ use free_list::{AllocError, FreeList, PageLayout, PageRange}; use hermit_sync::InterruptTicketMutex; +use crate::arch::x86_64::kernel::{get_limit, get_start, is_uefi}; use crate::arch::x86_64::mm::paging::{BasePageSize, PageSize}; use crate::arch::x86_64::mm::VirtAddr; use crate::mm; @@ -9,11 +10,15 @@ static KERNEL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); pub fn init() { - let range = PageRange::new( - mm::kernel_end_address().as_usize(), - kernel_heap_end().as_usize(), - ) - .unwrap(); + let range = if is_uefi() { + PageRange::new(get_start(), get_limit()).unwrap() + } else { + PageRange::new( + mm::kernel_end_address().as_usize(), + kernel_heap_end().as_usize(), + ) + .unwrap() + }; unsafe { KERNEL_FREE_LIST.lock().deallocate(range).unwrap(); } @@ -71,10 +76,10 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result= VirtAddr(mm::kernel_end_address().as_u64()), - "Virtual address {virtual_address:p} is not >= KERNEL_END_ADDRESS" - ); + // assert!( + // virtual_address >= VirtAddr(mm::kernel_end_address().as_u64()), + // "Virtual address {virtual_address:p} is not >= KERNEL_END_ADDRESS" + // ); assert!( virtual_address < kernel_heap_end(), "Virtual address {virtual_address:p} is not < kernel_heap_end()" diff --git a/src/drivers/virtio/env.rs b/src/drivers/virtio/env.rs index dbd4bd0e50..14641de1f1 100644 --- a/src/drivers/virtio/env.rs +++ b/src/drivers/virtio/env.rs @@ -218,16 +218,20 @@ pub mod pci { continue; } - let virtual_address = VirtMemAddr::from( - crate::mm::map( - PhysAddr::from(address), - size.try_into().unwrap(), - true, - true, - true, + let virtual_address = if !crate::kernel::is_uefi() { + VirtMemAddr::from( + crate::mm::map( + PhysAddr::from(address), + size.try_into().unwrap(), + true, + true, + true, + ) + .0, ) - .0, - ); + } else { + VirtMemAddr::from(address) + }; mapped_bars.push(VirtioPciBar::new( i.try_into().unwrap(), diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 38ffadf7d9..cb5593a0ef 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -92,22 +92,45 @@ pub(crate) fn init() { let has_2mib_pages = arch::processor::supports_2mib_pages(); //info!("reserved space {} KB", reserved_space >> 10); - - if total_memory_size() - < kernel_end_address().as_usize() - env::get_ram_address().as_usize() - + reserved_space - + LargePageSize::SIZE as usize - { - panic!("No enough memory available!"); - } + print_information(); let mut map_addr: VirtAddr; let mut map_size: usize; - let available_memory = (total_memory_size() - - (kernel_end_address().as_usize() - env::get_ram_address().as_usize()) - - reserved_space) - .align_down(LargePageSize::SIZE as usize); + //in the case of UEFI, the given memory is guaranteed free memory and the kernel is located before the given memory + let available_memory = if crate::arch::x86_64::kernel::is_uefi() { + (total_memory_size() - reserved_space).align_down(LargePageSize::SIZE as usize) + } else { + if total_memory_size() + < kernel_end_address().as_usize() - env::get_ram_address().as_usize() + + reserved_space + LargePageSize::SIZE as usize + { + panic!("Not enough memory available!"); + } + + (total_memory_size() + - (kernel_end_address().as_usize() - env::get_ram_address().as_usize()) + - reserved_space) + .align_down(LargePageSize::SIZE as usize) + }; + // if !crate::arch::x86_64::kernel::is_uefi() { + // if total_memory_size() + // < kernel_end_address().as_usize() - env::get_ram_address().as_usize() + // + reserved_space + LargePageSize::SIZE as usize + // { + // panic!("No enough memory available!"); + // } + + // let available_memory = (total_memory_size() + // - (kernel_end_address().as_usize() - env::get_ram_address().as_usize()) + // - reserved_space) + // .align_down(LargePageSize::SIZE as usize); + // } else { + // + + // let available_memory = + // (total_memory_size() - reserved_space).align_down(LargePageSize::SIZE as usize); + // } let heap_start_addr;