Merge pull request #857 from mkroening/late-heap

feat: initialize heap after paging

Cargo.toml

@@ -84,6 +84,8 @@ pflock = "0.2"
 qemu-exit = "3.0"
 rand_chacha = { version = "0.3", default-features = false }
 shell-words = { version = "1.1", default-features = false }
+smallvec = { version = "1", features = ["const_new"] }
+take-static = "0.1"
 talc = { version = "2" }
 time = { version = "0.3", default-features = false }
 zerocopy = { version = "0.7", features = ["derive"] }

src/arch/aarch64/kernel/core_local.rs

@@ -26,8 +26,12 @@ impl CoreLocal {
 	pub fn install() {
 		let core_id = CPU_ONLINE.load(Ordering::Relaxed);
 
-		let irq_statistics = &*Box::leak(Box::new(IrqStatistics::new()));
-		IRQ_COUNTERS.lock().insert(core_id, irq_statistics);
+		let irq_statistics = if core_id == 0 {
+			static FIRST_IRQ_STATISTICS: IrqStatistics = IrqStatistics::new();
+			&FIRST_IRQ_STATISTICS
+		} else {
+			&*Box::leak(Box::new(IrqStatistics::new()))
+		};
 
 		let this = Self {
 			this: ptr::null_mut(),
@@ -36,7 +40,15 @@ impl CoreLocal {
 			irq_statistics,
 			async_tasks: RefCell::new(Vec::new()),
 		};
-		let this = Box::leak(Box::new(this));
+		let this = if core_id == 0 {
+			take_static::take_static! {
+				static FIRST_CORE_LOCAL: Option<CoreLocal> = None;
+			}
+			FIRST_CORE_LOCAL.take().unwrap().insert(this)
+		} else {
+			this.add_irq_counter();
+			Box::leak(Box::new(this))
+		};
 		this.this = &*this;
 
 		unsafe {
@@ -52,6 +64,12 @@ impl CoreLocal {
 			&*raw
 		}
 	}
+
+	pub fn add_irq_counter(&self) {
+		IRQ_COUNTERS
+			.lock()
+			.insert(self.core_id, self.irq_statistics);
+	}
 }
 
 #[inline]

src/arch/aarch64/kernel/mod.rs

@@ -195,6 +195,7 @@ pub fn boot_processor_init() {
 
 	crate::mm::init();
 	crate::mm::print_information();
+	CoreLocal::get().add_irq_counter();
 	env::init();
 	interrupts::init();
 	interrupts::enable();

src/arch/aarch64/mm/physicalmem.rs

@@ -27,7 +27,7 @@ fn detect_from_limits() -> Result<(), ()> {
 		limit - mm::kernel_end_address().as_usize(),
 		Ordering::SeqCst,
 	);
-	PHYSICAL_FREE_LIST.lock().list.push_back(entry);
+	PHYSICAL_FREE_LIST.lock().push(entry);
 
 	Ok(())
 }

src/arch/aarch64/mm/virtualmem.rs

@@ -19,7 +19,7 @@ pub fn init() {
 		start: mm::kernel_end_address().as_usize(),
 		end: KERNEL_VIRTUAL_MEMORY_END.as_usize(),
 	};
-	KERNEL_FREE_LIST.lock().list.push_back(entry);
+	KERNEL_FREE_LIST.lock().push(entry);
 }
 
 pub fn allocate(size: usize) -> Result<VirtAddr, AllocError> {

src/arch/x86_64/kernel/core_local.rs

@@ -37,8 +37,12 @@ impl CoreLocal {
 
 		let core_id = CPU_ONLINE.load(Ordering::Relaxed);
 
-		let irq_statistics = &*Box::leak(Box::new(IrqStatistics::new()));
-		IRQ_COUNTERS.lock().insert(core_id, irq_statistics);
+		let irq_statistics = if core_id == 0 {
+			static FIRST_IRQ_STATISTICS: IrqStatistics = IrqStatistics::new();
+			&FIRST_IRQ_STATISTICS
+		} else {
+			&*Box::leak(Box::new(IrqStatistics::new()))
+		};
 
 		let this = Self {
 			this: ptr::null_mut(),
@@ -49,7 +53,15 @@ impl CoreLocal {
 			irq_statistics,
 			async_tasks: RefCell::new(Vec::new()),
 		};
-		let this = Box::leak(Box::new(this));
+		let this = if core_id == 0 {
+			take_static::take_static! {
+				static FIRST_CORE_LOCAL: Option<CoreLocal> = None;
+			}
+			FIRST_CORE_LOCAL.take().unwrap().insert(this)
+		} else {
+			this.add_irq_counter();
+			Box::leak(Box::new(this))
+		};
 		this.this = &*this;
 
 		GsBase::write(VirtAddr::from_ptr(this));
@@ -64,6 +76,12 @@ impl CoreLocal {
 			&*raw
 		}
 	}
+
+	pub fn add_irq_counter(&self) {
+		IRQ_COUNTERS
+			.lock()
+			.insert(self.core_id, self.irq_statistics);
+	}
 }
 
 pub(crate) fn core_id() -> CoreId {

src/arch/x86_64/kernel/mod.rs

@@ -213,6 +213,7 @@ pub fn boot_processor_init() {
 
 	crate::mm::init();
 	crate::mm::print_information();
+	CoreLocal::get().add_irq_counter();
 	env::init();
 	gdt::add_current_core();
 	interrupts::load_idt();

src/arch/x86_64/mm/physicalmem.rs

@@ -53,7 +53,7 @@ fn detect_from_multiboot_info() -> Result<(), ()> {
 			(m.base_address() + m.length() - start_address.as_u64()) as usize,
 			Ordering::SeqCst,
 		);
-		PHYSICAL_FREE_LIST.lock().list.push_back(entry);
+		PHYSICAL_FREE_LIST.lock().push(entry);
 	}
 
 	assert!(
@@ -73,17 +73,17 @@ fn detect_from_limits() -> Result<(), ()> {
 	// add gap for the APIC
 	if limit > KVM_32BIT_GAP_START {
 		let entry = FreeListEntry::new(mm::kernel_end_address().as_usize(), KVM_32BIT_GAP_START);
-		PHYSICAL_FREE_LIST.lock().list.push_back(entry);
+		PHYSICAL_FREE_LIST.lock().push(entry);
 		if limit > KVM_32BIT_GAP_START + KVM_32BIT_GAP_SIZE {
 			let entry = FreeListEntry::new(KVM_32BIT_GAP_START + KVM_32BIT_GAP_SIZE, limit);
-			PHYSICAL_FREE_LIST.lock().list.push_back(entry);
+			PHYSICAL_FREE_LIST.lock().push(entry);
 			TOTAL_MEMORY.store(limit - KVM_32BIT_GAP_SIZE, Ordering::SeqCst);
 		} else {
 			TOTAL_MEMORY.store(KVM_32BIT_GAP_START, Ordering::SeqCst);
 		}
 	} else {
 		let entry = FreeListEntry::new(mm::kernel_end_address().as_usize(), limit);
-		PHYSICAL_FREE_LIST.lock().list.push_back(entry);
+		PHYSICAL_FREE_LIST.lock().push(entry);
 		TOTAL_MEMORY.store(limit, Ordering::SeqCst);
 	}
 

src/arch/x86_64/mm/virtualmem.rs

@@ -15,7 +15,7 @@ pub fn init() {
 		mm::kernel_end_address().as_usize(),
 		kernel_heap_end().as_usize(),
 	);
-	KERNEL_FREE_LIST.lock().list.push_back(entry);
+	KERNEL_FREE_LIST.lock().push(entry);
 }
 
 pub fn allocate(size: usize) -> Result<VirtAddr, AllocError> {

src/lib.rs

@@ -299,13 +299,6 @@ fn synch_all_cores() {
 /// Entry Point of HermitCore for the Boot Processor
 #[cfg(target_os = "none")]
 fn boot_processor_main() -> ! {
-	let init_heap_start = env::get_base_address() + env::get_image_size();
-	let init_heap_len =
-		init_heap_start.align_up_to_large_page().as_usize() - init_heap_start.as_usize();
-	unsafe {
-		ALLOCATOR.init(init_heap_start.as_mut_ptr(), init_heap_len);
-	}
-
 	// Initialize the kernel and hardware.
 	arch::message_output_init();
 	unsafe {
@@ -326,11 +319,6 @@ fn boot_processor_main() -> ! {
 		env::get_tls_start(),
 		env::get_tls_memsz()
 	);
-	info!(
-		"Init heap: [0x{:p} - 0x{:p}]",
-		init_heap_start,
-		init_heap_start + init_heap_len
-	);
 	arch::boot_processor_init();
 	scheduler::add_current_core();
 

src/mm/allocator.rs

@@ -31,13 +31,6 @@ impl LockedAllocator {
 			self.0.talc().init(arena);
 		}
 	}
-
-	pub unsafe fn extend(&self, heap_bottom: *mut u8, heap_size: usize) {
-		let arena = Span::from_base_size(heap_bottom, heap_size);
-		unsafe {
-			self.0.talc().extend(arena);
-		}
-	}
 }
 
 /// To avoid false sharing, the global memory allocator align