From 1ec3c3a07951907ed83d516d22e19d357745dec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Mon, 29 Apr 2024 14:07:29 +0200 Subject: [PATCH 1/7] style(arch/mm): rename `alignment` variables to `align` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/aarch64/mm/physicalmem.rs | 14 +++++++------- src/arch/aarch64/mm/virtualmem.rs | 14 +++++++------- src/arch/riscv64/mm/physicalmem.rs | 14 +++++++------- src/arch/riscv64/mm/virtualmem.rs | 14 +++++++------- src/arch/x86_64/mm/physicalmem.rs | 14 +++++++------- src/arch/x86_64/mm/virtualmem.rs | 14 +++++++------- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/arch/aarch64/mm/physicalmem.rs b/src/arch/aarch64/mm/physicalmem.rs index 73f6a090a8..ea59141b2b 100644 --- a/src/arch/aarch64/mm/physicalmem.rs +++ b/src/arch/aarch64/mm/physicalmem.rs @@ -61,26 +61,26 @@ pub fn allocate(size: usize) -> Result { )) } -pub fn allocate_aligned(size: usize, alignment: usize) -> Result { +pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); - assert!(alignment > 0); + assert!(align > 0); assert_eq!( - size % alignment, + size % align, 0, - "Size {size:#X} is not a multiple of the given alignment {alignment:#X}" + "Size {size:#X} is not a multiple of the given alignment {align:#X}" ); assert_eq!( - alignment % BasePageSize::SIZE as usize, + align % BasePageSize::SIZE as usize, 0, "Alignment {:#X} is not a multiple of {:#X}", - alignment, + align, BasePageSize::SIZE ); Ok(PhysAddr( PHYSICAL_FREE_LIST .lock() - .allocate(size, Some(alignment))? + .allocate(size, Some(align))? .try_into() .unwrap(), )) diff --git a/src/arch/aarch64/mm/virtualmem.rs b/src/arch/aarch64/mm/virtualmem.rs index 77bce6be8b..700f0b105b 100644 --- a/src/arch/aarch64/mm/virtualmem.rs +++ b/src/arch/aarch64/mm/virtualmem.rs @@ -41,26 +41,26 @@ pub fn allocate(size: usize) -> Result { )) } -pub fn allocate_aligned(size: usize, alignment: usize) -> Result { +pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); - assert!(alignment > 0); + assert!(align > 0); assert_eq!( - size % alignment, + size % align, 0, - "Size {size:#X} is not a multiple of the given alignment {alignment:#X}" + "Size {size:#X} is not a multiple of the given alignment {align:#X}" ); assert_eq!( - alignment % BasePageSize::SIZE as usize, + align % BasePageSize::SIZE as usize, 0, "Alignment {:#X} is not a multiple of {:#X}", - alignment, + align, BasePageSize::SIZE ); Ok(VirtAddr( KERNEL_FREE_LIST .lock() - .allocate(size, Some(alignment))? + .allocate(size, Some(align))? .try_into() .unwrap(), )) diff --git a/src/arch/riscv64/mm/physicalmem.rs b/src/arch/riscv64/mm/physicalmem.rs index a3374c879a..67e7b61cbe 100644 --- a/src/arch/riscv64/mm/physicalmem.rs +++ b/src/arch/riscv64/mm/physicalmem.rs @@ -56,26 +56,26 @@ pub fn allocate(size: usize) -> Result { )) } -pub fn allocate_aligned(size: usize, alignment: usize) -> Result { +pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); - assert!(alignment > 0); + assert!(align > 0); assert_eq!( - size % alignment, + size % align, 0, - "Size {size:#X} is not a multiple of the given alignment {alignment:#X}" + "Size {size:#X} is not a multiple of the given alignment {align:#X}" ); assert_eq!( - alignment % BasePageSize::SIZE as usize, + align % BasePageSize::SIZE as usize, 0, "Alignment {:#X} is not a multiple of {:#X}", - alignment, + align, BasePageSize::SIZE as usize ); Ok(PhysAddr( PHYSICAL_FREE_LIST .lock() - .allocate(size, Some(alignment))? + .allocate(size, Some(align))? .try_into() .unwrap(), )) diff --git a/src/arch/riscv64/mm/virtualmem.rs b/src/arch/riscv64/mm/virtualmem.rs index e466e79cdc..974b8d37e5 100644 --- a/src/arch/riscv64/mm/virtualmem.rs +++ b/src/arch/riscv64/mm/virtualmem.rs @@ -45,26 +45,26 @@ pub fn allocate(size: usize) -> Result { )) } -pub fn allocate_aligned(size: usize, alignment: usize) -> Result { +pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); - assert!(alignment > 0); + assert!(align > 0); assert_eq!( - size % alignment, + size % align, 0, - "Size {size:#X} is not a multiple of the given alignment {alignment:#X}", + "Size {size:#X} is not a multiple of the given alignment {align:#X}", ); assert_eq!( - alignment % BasePageSize::SIZE as usize, + align % BasePageSize::SIZE as usize, 0, "Alignment {:#X} is not a multiple of {:#X}", - alignment, + align, BasePageSize::SIZE as usize ); Ok(VirtAddr( KERNEL_FREE_LIST .lock() - .allocate(size, Some(alignment))? + .allocate(size, Some(align))? .try_into() .unwrap(), )) diff --git a/src/arch/x86_64/mm/physicalmem.rs b/src/arch/x86_64/mm/physicalmem.rs index 9b46475d04..7cc2175396 100644 --- a/src/arch/x86_64/mm/physicalmem.rs +++ b/src/arch/x86_64/mm/physicalmem.rs @@ -171,26 +171,26 @@ unsafe impl FrameAllocator for Frame } } -pub fn allocate_aligned(size: usize, alignment: usize) -> Result { +pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); - assert!(alignment > 0); + assert!(align > 0); assert_eq!( - size % alignment, + size % align, 0, - "Size {size:#X} is not a multiple of the given alignment {alignment:#X}" + "Size {size:#X} is not a multiple of the given alignment {align:#X}" ); assert_eq!( - alignment % BasePageSize::SIZE as usize, + align % BasePageSize::SIZE as usize, 0, "Alignment {:#X} is not a multiple of {:#X}", - alignment, + align, BasePageSize::SIZE ); Ok(PhysAddr( PHYSICAL_FREE_LIST .lock() - .allocate(size, Some(alignment))? + .allocate(size, Some(align))? .try_into() .unwrap(), )) diff --git a/src/arch/x86_64/mm/virtualmem.rs b/src/arch/x86_64/mm/virtualmem.rs index 3b751cedc6..63a1426cdc 100644 --- a/src/arch/x86_64/mm/virtualmem.rs +++ b/src/arch/x86_64/mm/virtualmem.rs @@ -38,26 +38,26 @@ pub fn allocate(size: usize) -> Result { } #[cfg(not(feature = "newlib"))] -pub fn allocate_aligned(size: usize, alignment: usize) -> Result { +pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); - assert!(alignment > 0); + assert!(align > 0); assert_eq!( - size % alignment, + size % align, 0, - "Size {size:#X} is not a multiple of the given alignment {alignment:#X}" + "Size {size:#X} is not a multiple of the given alignment {align:#X}" ); assert_eq!( - alignment % BasePageSize::SIZE as usize, + align % BasePageSize::SIZE as usize, 0, "Alignment {:#X} is not a multiple of {:#X}", - alignment, + align, BasePageSize::SIZE ); Ok(VirtAddr( KERNEL_FREE_LIST .lock() - .allocate(size, Some(alignment))? + .allocate(size, Some(align))? .try_into() .unwrap(), )) From c2211555f16e4c13c9bb04737bd099f763653b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 7 May 2024 17:40:10 +0200 Subject: [PATCH 2/7] style(mm): rename `sz` variables to `size` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/x86_64/kernel/gdt.rs | 6 +++--- src/arch/x86_64/mm/mod.rs | 4 ++-- src/mm/mod.rs | 16 ++++++++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/arch/x86_64/kernel/gdt.rs b/src/arch/x86_64/kernel/gdt.rs index fbb0a6d4ce..a76d882e7d 100644 --- a/src/arch/x86_64/kernel/gdt.rs +++ b/src/arch/x86_64/kernel/gdt.rs @@ -43,16 +43,16 @@ pub fn add_current_core() { // Allocate all ISTs for this core. // Every task later gets its own IST, so the IST allocated here is only used by the Idle task. for i in 0..IST_ENTRIES { - let sz = if i == 0 { + let size = if i == 0 { IST_SIZE } else { BasePageSize::SIZE as usize }; - let layout = Layout::from_size_align(sz, BasePageSize::SIZE as usize).unwrap(); + let layout = Layout::from_size_align(size, BasePageSize::SIZE as usize).unwrap(); let ist = unsafe { alloc(layout) }; assert!(!ist.is_null()); - let ist_start = unsafe { ist.add(sz - TaskStacks::MARKER_SIZE) }; + let ist_start = unsafe { ist.add(size - TaskStacks::MARKER_SIZE) }; tss.interrupt_stack_table[i] = VirtAddr::from_ptr(ist_start); } diff --git a/src/arch/x86_64/mm/mod.rs b/src/arch/x86_64/mm/mod.rs index b4e606ee09..2cdc130bcc 100644 --- a/src/arch/x86_64/mm/mod.rs +++ b/src/arch/x86_64/mm/mod.rs @@ -21,9 +21,9 @@ impl multiboot::information::MemoryManagement for MultibootMemory { unsafe fn paddr_to_slice( &self, p: multiboot::information::PAddr, - sz: usize, + size: usize, ) -> Option<&'static [u8]> { - unsafe { Some(slice::from_raw_parts(p as _, sz)) } + unsafe { Some(slice::from_raw_parts(p as _, size)) } } unsafe fn allocate( diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 95c985898c..a7ee74d1d5 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -314,8 +314,8 @@ pub(crate) fn print_information() { } /// Soft-deprecated in favor of `DeviceAlloc` -pub(crate) fn allocate(sz: usize, no_execution: bool) -> VirtAddr { - let size = sz.align_up(BasePageSize::SIZE as usize); +pub(crate) fn allocate(size: usize, no_execution: bool) -> VirtAddr { + let size = size.align_up(BasePageSize::SIZE as usize); let physical_address = arch::mm::physicalmem::allocate(size).unwrap(); let virtual_address = arch::mm::virtualmem::allocate(size).unwrap(); @@ -331,8 +331,8 @@ pub(crate) fn allocate(sz: usize, no_execution: bool) -> VirtAddr { } /// Soft-deprecated in favor of `DeviceAlloc` -pub(crate) fn deallocate(virtual_address: VirtAddr, sz: usize) { - let size = sz.align_up(BasePageSize::SIZE as usize); +pub(crate) fn deallocate(virtual_address: VirtAddr, size: usize) { + let size = size.align_up(BasePageSize::SIZE as usize); if let Some(phys_addr) = arch::mm::paging::virtual_to_physical(virtual_address) { arch::mm::paging::unmap::( @@ -353,12 +353,12 @@ pub(crate) fn deallocate(virtual_address: VirtAddr, sz: usize) { #[cfg(feature = "pci")] pub(crate) fn map( physical_address: PhysAddr, - sz: usize, + size: usize, writable: bool, no_execution: bool, no_cache: bool, ) -> VirtAddr { - let size = sz.align_up(BasePageSize::SIZE as usize); + let size = size.align_up(BasePageSize::SIZE as usize); let count = size / BasePageSize::SIZE as usize; let mut flags = PageTableEntryFlags::empty(); @@ -381,8 +381,8 @@ pub(crate) fn map( #[allow(dead_code)] /// unmaps virtual address, without 'freeing' physical memory it is mapped to! -pub(crate) fn unmap(virtual_address: VirtAddr, sz: usize) { - let size = sz.align_up(BasePageSize::SIZE as usize); +pub(crate) fn unmap(virtual_address: VirtAddr, size: usize) { + let size = size.align_up(BasePageSize::SIZE as usize); if arch::mm::paging::virtual_to_physical(virtual_address).is_some() { arch::mm::paging::unmap::( From c2d6abc8e6350dced02b1625ec49089fff9c474d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 30 Apr 2024 00:13:55 +0200 Subject: [PATCH 3/7] refactor(x86_64): migrate to free-list crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- Cargo.lock | 21 ++++++++- Cargo.toml | 1 + src/arch/x86_64/mm/physicalmem.rs | 76 +++++++++++++++++++------------ src/arch/x86_64/mm/virtualmem.rs | 38 ++++++++++------ src/mm/mod.rs | 1 + 5 files changed, 90 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7efd309b20..0be5ad125d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d39c09fbfba977f4842e1f6bdc48b979112b64f8886993a34e051bc5f3c5c288" +[[package]] +name = "align-address" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6c08a67736554282858203cd9b7ff53cf55f54c34e85689962748a350cbf0" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -456,6 +462,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "free-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d140bfddcc22e1f83bbc5d6930ec5713c62489cdd94b86c92852edfc1db96f2b" +dependencies = [ + "align-address 0.3.0", + "smallvec", +] + [[package]] name = "generic_once_cell" version = "0.1.1" @@ -530,7 +546,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa8da8546263459c84a24582eacf3b7b6d818a12f606e920bcfe5485235dac63" dependencies = [ - "align-address", + "align-address 0.1.0", "time", ] @@ -540,7 +556,7 @@ version = "0.7.0" dependencies = [ "aarch64", "ahash", - "align-address", + "align-address 0.1.0", "anstyle", "anyhow", "arm-gic", @@ -554,6 +570,7 @@ dependencies = [ "dyn-clone", "fdt", "float-cmp", + "free-list", "hashbrown", "hermit-dtb", "hermit-entry", diff --git a/Cargo.toml b/Cargo.toml index 6fc7dc9ee6..1ace093de3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ cfg-if = "1" crossbeam-utils = { version = "0.8", default-features = false } dyn-clone = "1.0" fdt = "0.1" +free-list = "0.3" hashbrown = { version = "0.14", default-features = false } hermit-entry = { version = "0.10", features = ["kernel"] } hermit-sync = "0.1" diff --git a/src/arch/x86_64/mm/physicalmem.rs b/src/arch/x86_64/mm/physicalmem.rs index 7cc2175396..ea7fface8e 100644 --- a/src/arch/x86_64/mm/physicalmem.rs +++ b/src/arch/x86_64/mm/physicalmem.rs @@ -1,7 +1,7 @@ -use core::alloc::AllocError; use core::sync::atomic::{AtomicUsize, Ordering}; use ::x86_64::structures::paging::{FrameAllocator, PhysFrame}; +use free_list::{AllocError, FreeList, PageLayout, PageRange}; use hermit_sync::InterruptTicketMutex; use multiboot::information::{MemoryType, Multiboot}; @@ -9,9 +9,8 @@ use crate::arch::x86_64::kernel::{get_fdt, get_limit, get_mbinfo}; use crate::arch::x86_64::mm::paging::{BasePageSize, PageSize}; use crate::arch::x86_64::mm::{MultibootMemory, PhysAddr, VirtAddr}; use crate::mm; -use crate::mm::freelist::{FreeList, FreeListEntry}; -static PHYSICAL_FREE_LIST: InterruptTicketMutex = +static PHYSICAL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); @@ -45,12 +44,14 @@ fn detect_from_fdt() -> Result<(), ()> { VirtAddr(start_address) }; - let entry = FreeListEntry::new(start_address.as_usize(), end_address as usize); + let range = PageRange::new(start_address.as_usize(), end_address as usize).unwrap(); let _ = TOTAL_MEMORY.fetch_add( (end_address - start_address.as_u64()) as usize, Ordering::SeqCst, ); - PHYSICAL_FREE_LIST.lock().push(entry); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } } assert!( @@ -84,15 +85,18 @@ fn detect_from_multiboot_info() -> Result<(), ()> { VirtAddr(m.base_address()) }; - let entry = FreeListEntry::new( + let range = PageRange::new( start_address.as_usize(), (m.base_address() + m.length()) as usize, - ); + ) + .unwrap(); let _ = TOTAL_MEMORY.fetch_add( (m.base_address() + m.length() - start_address.as_u64()) as usize, Ordering::SeqCst, ); - PHYSICAL_FREE_LIST.lock().push(entry); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } } assert!( @@ -111,18 +115,25 @@ fn detect_from_limits() -> Result<(), ()> { // add gap for the APIC if limit > KVM_32BIT_GAP_START { - let entry = FreeListEntry::new(mm::kernel_end_address().as_usize(), KVM_32BIT_GAP_START); - PHYSICAL_FREE_LIST.lock().push(entry); + let range = + PageRange::new(mm::kernel_end_address().as_usize(), KVM_32BIT_GAP_START).unwrap(); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } if limit > KVM_32BIT_GAP_START + KVM_32BIT_GAP_SIZE { - let entry = FreeListEntry::new(KVM_32BIT_GAP_START + KVM_32BIT_GAP_SIZE, limit); - PHYSICAL_FREE_LIST.lock().push(entry); + let range = PageRange::new(KVM_32BIT_GAP_START + KVM_32BIT_GAP_SIZE, limit).unwrap(); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } TOTAL_MEMORY.store(limit - KVM_32BIT_GAP_SIZE, Ordering::SeqCst); } else { TOTAL_MEMORY.store(KVM_32BIT_GAP_START, Ordering::SeqCst); } } else { - let entry = FreeListEntry::new(mm::kernel_end_address().as_usize(), limit); - PHYSICAL_FREE_LIST.lock().push(entry); + let range = PageRange::new(mm::kernel_end_address().as_usize(), limit).unwrap(); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } TOTAL_MEMORY.store(limit, Ordering::SeqCst); } @@ -150,10 +161,13 @@ pub fn allocate(size: usize) -> Result { BasePageSize::SIZE ); + let layout = PageLayout::from_size(size).unwrap(); + Ok(PhysAddr( PHYSICAL_FREE_LIST .lock() - .allocate(size, None)? + .allocate(layout)? + .start() .try_into() .unwrap(), )) @@ -163,10 +177,8 @@ pub struct FrameAlloc; unsafe impl FrameAllocator for FrameAlloc { fn allocate_frame(&mut self) -> Option> { - let addr = PHYSICAL_FREE_LIST - .lock() - .allocate(S::SIZE as usize, Some(S::SIZE as usize)) - .ok()? as u64; + let layout = PageLayout::from_size_align(S::SIZE as usize, S::SIZE as usize).unwrap(); + let addr = PHYSICAL_FREE_LIST.lock().allocate(layout).ok()?.start() as u64; Some(PhysFrame::from_start_address(x86_64::PhysAddr::new(addr)).unwrap()) } } @@ -187,10 +199,13 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result = +static KERNEL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); pub fn init() { - let entry = FreeListEntry::new( + let range = PageRange::new( mm::kernel_end_address().as_usize(), kernel_heap_end().as_usize(), - ); - KERNEL_FREE_LIST.lock().push(entry); + ) + .unwrap(); + unsafe { + KERNEL_FREE_LIST.lock().deallocate(range).unwrap(); + } } pub fn allocate(size: usize) -> Result { @@ -28,10 +29,13 @@ pub fn allocate(size: usize) -> Result { BasePageSize::SIZE ); + let layout = PageLayout::from_size(size).unwrap(); + Ok(VirtAddr( KERNEL_FREE_LIST .lock() - .allocate(size, None)? + .allocate(layout)? + .start() .try_into() .unwrap(), )) @@ -54,10 +58,13 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result Date: Tue, 30 Apr 2024 12:25:56 +0200 Subject: [PATCH 4/7] fix(x86_64/mm): remove `FrameAlloc` ZST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- Cargo.lock | 1 + Cargo.toml | 1 + src/arch/x86_64/mm/paging.rs | 7 +++++-- src/arch/x86_64/mm/physicalmem.rs | 13 +------------ 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0be5ad125d..f0063e655a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -470,6 +470,7 @@ checksum = "d140bfddcc22e1f83bbc5d6930ec5713c62489cdd94b86c92852edfc1db96f2b" dependencies = [ "align-address 0.3.0", "smallvec", + "x86_64 0.15.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1ace093de3..c4eac5e24d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,7 @@ features = [ ] [target.'cfg(target_arch = "x86_64")'.dependencies] +free-list = { version = "0.3", features = ["x86_64"] } multiboot = "0.8" uart_16550 = "0.3" x86 = { version = "0.52", default-features = false } diff --git a/src/arch/x86_64/mm/paging.rs b/src/arch/x86_64/mm/paging.rs index 62020f210a..8e80da762b 100644 --- a/src/arch/x86_64/mm/paging.rs +++ b/src/arch/x86_64/mm/paging.rs @@ -158,6 +158,7 @@ pub fn map( #[cfg(feature = "smp")] let mut ipi_tlb_flush = false; + let mut frame_allocator = physicalmem::PHYSICAL_FREE_LIST.lock(); for (page, frame) in pages.zip(frames) { unsafe { // TODO: Require explicit unmaps @@ -170,11 +171,12 @@ pub fn map( debug!("Had to unmap page {page:?} before mapping."); } recursive_page_table() - .map_to(page, frame, flags, &mut physicalmem::FrameAlloc) + .map_to(page, frame, flags, &mut *frame_allocator) .unwrap() .flush(); } } + drop(frame_allocator); #[cfg(feature = "smp")] if ipi_tlb_flush { @@ -218,12 +220,13 @@ where frame.start_address() ); + let mut frame_allocator = physicalmem::PHYSICAL_FREE_LIST.lock(); unsafe { recursive_page_table() .identity_map( frame, PageTableEntryFlags::PRESENT | PageTableEntryFlags::NO_EXECUTE, - &mut physicalmem::FrameAlloc, + &mut *frame_allocator, ) .unwrap() .flush(); diff --git a/src/arch/x86_64/mm/physicalmem.rs b/src/arch/x86_64/mm/physicalmem.rs index ea7fface8e..b410a5fc7d 100644 --- a/src/arch/x86_64/mm/physicalmem.rs +++ b/src/arch/x86_64/mm/physicalmem.rs @@ -1,6 +1,5 @@ use core::sync::atomic::{AtomicUsize, Ordering}; -use ::x86_64::structures::paging::{FrameAllocator, PhysFrame}; use free_list::{AllocError, FreeList, PageLayout, PageRange}; use hermit_sync::InterruptTicketMutex; use multiboot::information::{MemoryType, Multiboot}; @@ -10,7 +9,7 @@ use crate::arch::x86_64::mm::paging::{BasePageSize, PageSize}; use crate::arch::x86_64::mm::{MultibootMemory, PhysAddr, VirtAddr}; use crate::mm; -static PHYSICAL_FREE_LIST: InterruptTicketMutex> = +pub static PHYSICAL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); @@ -173,16 +172,6 @@ pub fn allocate(size: usize) -> Result { )) } -pub struct FrameAlloc; - -unsafe impl FrameAllocator for FrameAlloc { - fn allocate_frame(&mut self) -> Option> { - let layout = PageLayout::from_size_align(S::SIZE as usize, S::SIZE as usize).unwrap(); - let addr = PHYSICAL_FREE_LIST.lock().allocate(layout).ok()?.start() as u64; - Some(PhysFrame::from_start_address(x86_64::PhysAddr::new(addr)).unwrap()) - } -} - pub fn allocate_aligned(size: usize, align: usize) -> Result { assert!(size > 0); assert!(align > 0); From 05d26132b2d61e0c5aed8e9de7e06065a91b32de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Wed, 8 May 2024 16:27:36 +0200 Subject: [PATCH 5/7] refactor(aarch64): migrate to free-list crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/aarch64/mm/physicalmem.rs | 37 ++++++++++++++------------ src/arch/aarch64/mm/virtualmem.rs | 42 ++++++++++++++++++------------ src/mm/mod.rs | 2 +- 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/src/arch/aarch64/mm/physicalmem.rs b/src/arch/aarch64/mm/physicalmem.rs index ea59141b2b..8d8d958996 100644 --- a/src/arch/aarch64/mm/physicalmem.rs +++ b/src/arch/aarch64/mm/physicalmem.rs @@ -1,15 +1,14 @@ -use core::alloc::AllocError; use core::sync::atomic::{AtomicUsize, Ordering}; +use free_list::{AllocError, FreeList, PageLayout, PageRange}; use hermit_sync::InterruptTicketMutex; use crate::arch::aarch64::kernel::get_limit; use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize}; use crate::arch::aarch64::mm::PhysAddr; use crate::mm; -use crate::mm::freelist::{FreeList, FreeListEntry}; -static PHYSICAL_FREE_LIST: InterruptTicketMutex = +static PHYSICAL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); @@ -19,15 +18,14 @@ fn detect_from_limits() -> Result<(), ()> { return Err(()); } - let entry = FreeListEntry { - start: mm::kernel_end_address().as_usize(), - end: limit, - }; + let range = PageRange::new(mm::kernel_end_address().as_usize(), limit).unwrap(); TOTAL_MEMORY.store( limit - mm::kernel_end_address().as_usize(), Ordering::SeqCst, ); - PHYSICAL_FREE_LIST.lock().push(entry); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } Ok(()) } @@ -52,10 +50,13 @@ pub fn allocate(size: usize) -> Result { BasePageSize::SIZE ); + let layout = PageLayout::from_size(size).unwrap(); + Ok(PhysAddr( PHYSICAL_FREE_LIST .lock() - .allocate(size, None)? + .allocate(layout)? + .start() .try_into() .unwrap(), )) @@ -77,10 +78,13 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result = +static KERNEL_FREE_LIST: InterruptTicketMutex> = InterruptTicketMutex::new(FreeList::new()); /// End of the virtual memory address space reserved for kernel memory (4 GiB). @@ -15,11 +13,14 @@ static KERNEL_FREE_LIST: InterruptTicketMutex = const KERNEL_VIRTUAL_MEMORY_END: VirtAddr = VirtAddr(0x1_0000_0000); pub fn init() { - let entry = FreeListEntry { - start: mm::kernel_end_address().as_usize(), - end: KERNEL_VIRTUAL_MEMORY_END.as_usize(), - }; - KERNEL_FREE_LIST.lock().push(entry); + let range = PageRange::new( + mm::kernel_end_address().as_usize(), + KERNEL_VIRTUAL_MEMORY_END.as_usize(), + ) + .unwrap(); + unsafe { + KERNEL_FREE_LIST.lock().deallocate(range).unwrap(); + } } pub fn allocate(size: usize) -> Result { @@ -32,10 +33,13 @@ pub fn allocate(size: usize) -> Result { BasePageSize::SIZE ); + let layout = PageLayout::from_size(size).unwrap(); + Ok(VirtAddr( KERNEL_FREE_LIST .lock() - .allocate(size, None)? + .allocate(layout)? + .start() .try_into() .unwrap(), )) @@ -57,10 +61,13 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result Date: Wed, 8 May 2024 16:52:24 +0200 Subject: [PATCH 6/7] refactor(riscv64): migrate to free-list crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/riscv64/mm/physicalmem.rs | 38 +++++++++++++++++---------- src/arch/riscv64/mm/virtualmem.rs | 42 ++++++++++++++++++------------ src/logging.rs | 2 ++ src/mm/mod.rs | 6 ++++- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/arch/riscv64/mm/physicalmem.rs b/src/arch/riscv64/mm/physicalmem.rs index 67e7b61cbe..89be2606cb 100644 --- a/src/arch/riscv64/mm/physicalmem.rs +++ b/src/arch/riscv64/mm/physicalmem.rs @@ -1,16 +1,16 @@ -use core::alloc::AllocError; use core::convert::TryInto; use core::sync::atomic::{AtomicUsize, Ordering}; +use free_list::{AllocError, FreeList, PageLayout, PageRange}; use hermit_sync::InterruptSpinMutex; use crate::arch::riscv64::kernel::{get_limit, get_ram_address}; use crate::arch::riscv64::mm::paging::{BasePageSize, PageSize}; use crate::arch::riscv64::mm::PhysAddr; use crate::mm; -use crate::mm::freelist::{FreeList, FreeListEntry}; -static PHYSICAL_FREE_LIST: InterruptSpinMutex = InterruptSpinMutex::new(FreeList::new()); +static PHYSICAL_FREE_LIST: InterruptSpinMutex> = + InterruptSpinMutex::new(FreeList::new()); static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); fn detect_from_limits() -> Result<(), ()> { @@ -19,11 +19,14 @@ fn detect_from_limits() -> Result<(), ()> { return Err(()); } - let entry = FreeListEntry::new( + let range = PageRange::new( mm::kernel_end_address().as_usize(), get_ram_address().as_usize() + limit, - ); - PHYSICAL_FREE_LIST.lock().push(entry); + ) + .unwrap(); + unsafe { + PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap(); + } TOTAL_MEMORY.store(limit, Ordering::SeqCst); Ok(()) @@ -47,10 +50,13 @@ pub fn allocate(size: usize) -> Result { BasePageSize::SIZE as usize ); + let layout = PageLayout::from_size(size).unwrap(); + Ok(PhysAddr( PHYSICAL_FREE_LIST .lock() - .allocate(size, None)? + .allocate(layout)? + .start() .try_into() .unwrap(), )) @@ -72,10 +78,13 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result = InterruptSpinMutex::new(FreeList::new()); +static KERNEL_FREE_LIST: InterruptSpinMutex> = + InterruptSpinMutex::new(FreeList::new()); /// End of the virtual memory address space reserved for kernel memory (256 GiB). /// This also marks the start of the virtual memory address space reserved for the task heap. const KERNEL_VIRTUAL_MEMORY_END: VirtAddr = VirtAddr(0x4000000000); pub fn init() { - let entry = FreeListEntry { - start: (get_ram_address() + PhysAddr(physicalmem::total_memory_size() as u64)) + let range = PageRange::new( + (get_ram_address() + PhysAddr(physicalmem::total_memory_size() as u64)) .as_usize() .align_up(HugePageSize::SIZE as usize), - end: KERNEL_VIRTUAL_MEMORY_END.as_usize(), - }; - KERNEL_FREE_LIST.lock().push(entry); + KERNEL_VIRTUAL_MEMORY_END.as_usize(), + ) + .unwrap(); + unsafe { + KERNEL_FREE_LIST.lock().deallocate(range).unwrap(); + } } pub fn allocate(size: usize) -> Result { @@ -36,10 +39,13 @@ pub fn allocate(size: usize) -> Result { BasePageSize::SIZE as usize ); + let layout = PageLayout::from_size(size).unwrap(); + Ok(VirtAddr( KERNEL_FREE_LIST .lock() - .allocate(size, None)? + .allocate(layout)? + .start() .try_into() .unwrap(), )) @@ -61,10 +67,13 @@ pub fn allocate_aligned(size: usize, align: usize) -> Result (::log::info!("{:25}{}", concat!($str, ":"), format_args!($($arg)+))); } +#[cfg(any(not(target_arch = "riscv64"), feature = "pci", feature = "tcp"))] macro_rules! infofooter { () => {{ ::log::info!("{:=^70}", '='); diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 574c6f0f4c..34a5f0c33d 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -1,6 +1,10 @@ pub mod allocator; pub mod device_alloc; -#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +#[cfg(not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "riscv64" +)))] pub mod freelist; use core::mem; From f285c4a45f0a45f55f1db4dc3308ae1557b92340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Wed, 8 May 2024 16:58:08 +0200 Subject: [PATCH 7/7] fix(mm): remove freelist module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/mm/freelist.rs | 229 --------------------------------------------- src/mm/mod.rs | 6 -- 2 files changed, 235 deletions(-) delete mode 100644 src/mm/freelist.rs diff --git a/src/mm/freelist.rs b/src/mm/freelist.rs deleted file mode 100644 index 2b11721976..0000000000 --- a/src/mm/freelist.rs +++ /dev/null @@ -1,229 +0,0 @@ -use core::alloc::AllocError; -use core::cmp::Ordering; - -use align_address::Align; -use smallvec::SmallVec; - -#[derive(Debug)] -pub struct FreeListEntry { - pub start: usize, - pub end: usize, -} - -impl FreeListEntry { - pub const fn new(start: usize, end: usize) -> Self { - FreeListEntry { start, end } - } -} - -#[derive(Debug)] -pub struct FreeList { - list: SmallVec<[FreeListEntry; 16]>, -} - -impl FreeList { - pub const fn new() -> Self { - Self { - list: SmallVec::new_const(), - } - } - - pub fn push(&mut self, entry: FreeListEntry) { - self.list.push(entry); - } - - pub fn allocate(&mut self, size: usize, alignment: Option) -> Result { - trace!("Allocating {} bytes from Free List {self:p}", size); - - let new_size = if let Some(align) = alignment { - size + align - } else { - size - }; - - // Find a region in the Free List that has at least the requested size. - for (i, node) in self.list.iter_mut().enumerate() { - let (region_start, region_size) = (node.start, node.end - node.start); - - match region_size.cmp(&new_size) { - Ordering::Greater => { - // We have found a region that is larger than the requested size. - // Return the address to the beginning of that region and shrink the region by that size. - if let Some(align) = alignment { - let new_addr = region_start.align_up(align); - node.start += size + (new_addr - region_start); - if new_addr != region_start { - let new_entry = FreeListEntry::new(region_start, new_addr); - self.list.insert(i, new_entry); - } - return Ok(new_addr); - } else { - node.start += size; - return Ok(region_start); - } - } - Ordering::Equal => { - // We have found a region that has exactly the requested size. - // Return the address to the beginning of that region and move the node into the pool for deletion or reuse. - if let Some(align) = alignment { - let new_addr = region_start.align_up(align); - if new_addr != region_start { - node.end = new_addr; - } - return Ok(new_addr); - } else { - self.list.remove(i); - return Ok(region_start); - } - } - Ordering::Less => {} - } - } - - Err(AllocError) - } - - #[cfg(all(target_arch = "x86_64", not(feature = "pci")))] - pub fn reserve(&mut self, address: usize, size: usize) -> Result<(), AllocError> { - trace!( - "Try to reserve {} bytes at {:#X} from Free List {self:p}", - size, - address - ); - - // Find a region in the Free List that has at least the requested size. - for (i, node) in self.list.iter_mut().enumerate() { - let (region_start, region_size) = (node.start, node.end - node.start); - - if address > region_start && address + size < region_start + region_size { - node.start = address + size; - let new_entry = FreeListEntry::new(region_start, address); - self.list.insert(i, new_entry); - return Ok(()); - } else if address > region_start && address + size == region_start + region_size { - node.start = address + size; - return Ok(()); - } else if address == region_start && address + size < region_start + region_size { - node.start = region_start + size; - return Ok(()); - } - } - - Err(AllocError) - } - - pub fn deallocate(&mut self, address: usize, size: usize) { - trace!( - "Deallocating {} bytes at {:#X} from Free List {self:p}", - size, - address - ); - - let end = address + size; - - for (i, node) in self.list.iter_mut().enumerate() { - let (region_start, region_end) = (node.start, node.end); - - if region_start == end { - // The deallocated memory extends this free memory region to the left. - node.start = address; - - // Check if it can even reunite with the previous region. - if i > 0 { - if let Some(prev_node) = self.list.get_mut(i - 1) { - let prev_region_end = prev_node.end; - - if prev_region_end == address { - // It can reunite, so let the current region span over the reunited region and move the duplicate node - // into the pool for deletion or reuse. - prev_node.end = region_end; - self.list.remove(i); - } - } - } - - return; - } else if region_end == address { - node.end = end; - - // Check if it can even reunite with the next region. - if let Some(next_node) = self.list.get_mut(i + 1) { - let next_region_start = next_node.start; - - if next_region_start == end { - // It can reunite, so let the current region span over the reunited region and move the duplicate node - // into the pool for deletion or reuse. - next_node.start = region_start; - self.list.remove(i); - } - } - - return; - } else if end < region_start { - // The deallocated memory does not extend any memory region and needs an own entry in the Free List. - // Get that entry from the node pool. - // We search the list from low to high addresses and insert us before the first entry that has a - // higher address than us. - let new_entry = FreeListEntry::new(address, end); - self.list.insert(i, new_entry); - return; - } - } - - // We could not find an entry with a higher address than us. - // So we become the new last entry in the list. Get that entry from the node pool. - let new_element = FreeListEntry::new(address, end); - self.push(new_element); - } - - pub fn print_information(&self, header: &str) { - infoheader!(header); - - for node in self.list.iter() { - info!("{:#016X} - {:#016X}", node.start, node.end); - } - - infofooter!(); - } -} - -#[cfg(all(test, not(target_os = "none")))] -mod tests { - use super::*; - - #[test] - fn allocate() { - let mut freelist = FreeList::new(); - let entry = FreeListEntry::new(0x10000, 0x100000); - - freelist.push(entry); - let addr = freelist.allocate(0x1000, None); - - assert_eq!(addr.unwrap(), 0x10000); - - for node in &freelist.list { - assert_eq!(node.start, 0x11000); - assert_eq!(node.end, 0x100000); - } - - let addr = freelist.allocate(0x1000, Some(0x2000)); - let mut iter = freelist.list.iter(); - assert_eq!(iter.next().unwrap().start, 0x11000); - assert_eq!(iter.next().unwrap().start, 0x13000); - } - - #[test] - fn deallocate() { - let mut freelist = FreeList::new(); - let entry = FreeListEntry::new(0x10000, 0x100000); - - freelist.push(entry); - let addr = freelist.allocate(0x1000, None); - freelist.deallocate(addr.unwrap(), 0x1000); - - for node in &freelist.list { - assert_eq!(node.start, 0x10000); - assert_eq!(node.end, 0x100000); - } - } -} diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 34a5f0c33d..38ffadf7d9 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -1,11 +1,5 @@ pub mod allocator; pub mod device_alloc; -#[cfg(not(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "riscv64" -)))] -pub mod freelist; use core::mem; use core::ops::Range;