From ceeb57dec5e7212cb7826278dfcb8ee76ae02a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Wed, 13 Sep 2023 14:46:06 +0200 Subject: [PATCH 1/7] style(scheduler): inline closure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/scheduler/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index f937515c20..e55032ce60 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -174,7 +174,7 @@ impl PerCoreScheduler { /// Terminate the current task on the current core. pub fn exit(&mut self, exit_code: i32) -> ! { - let closure = || { + without_interrupts(|| { // Get the current task. let mut current_task_borrowed = self.current_task.borrow_mut(); assert_ne!( @@ -200,9 +200,7 @@ impl PerCoreScheduler { self.custom_wakeup(task); } } - }; - - without_interrupts(closure); + }); self.reschedule(); unreachable!() From 64a9620556f579c8d696dc11c21a2695a321f137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 12 Sep 2023 12:39:19 +0200 Subject: [PATCH 2/7] refactor(scheduler): move scheduler input to core-local storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/aarch64/kernel/core_local.rs | 10 ++++++++++ src/arch/x86_64/kernel/core_local.rs | 9 +++++++++ src/scheduler/mod.rs | 17 ++++++----------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/arch/aarch64/kernel/core_local.rs b/src/arch/aarch64/kernel/core_local.rs index 71c917d9f5..a700972923 100644 --- a/src/arch/aarch64/kernel/core_local.rs +++ b/src/arch/aarch64/kernel/core_local.rs @@ -5,9 +5,14 @@ use core::cell::{Cell, RefCell, RefMut}; use core::ptr; use core::sync::atomic::Ordering; +#[cfg(feature = "smp")] +use hermit_sync::InterruptTicketMutex; + use super::interrupts::{IrqStatistics, IRQ_COUNTERS}; use super::CPU_ONLINE; use crate::executor::task::AsyncTask; +#[cfg(feature = "smp")] +use crate::scheduler::SchedulerInput; use crate::scheduler::{CoreId, PerCoreScheduler}; pub(crate) struct CoreLocal { @@ -20,6 +25,9 @@ pub(crate) struct CoreLocal { irq_statistics: &'static IrqStatistics, /// Queue of async tasks async_tasks: RefCell>, + /// Queues to handle incoming requests from the other cores + #[cfg(feature = "smp")] + pub scheduler_input: InterruptTicketMutex, } impl CoreLocal { @@ -39,6 +47,8 @@ impl CoreLocal { scheduler: Cell::new(ptr::null_mut()), irq_statistics, async_tasks: RefCell::new(Vec::new()), + #[cfg(feature = "smp")] + scheduler_input: InterruptTicketMutex::new(SchedulerInput::new()), }; let this = if core_id == 0 { take_static::take_static! { diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index 323b396a48..bf6210f5fa 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -5,6 +5,8 @@ use core::cell::{Cell, RefCell, RefMut}; use core::ptr; use core::sync::atomic::Ordering; +#[cfg(feature = "smp")] +use hermit_sync::InterruptTicketMutex; use x86_64::registers::model_specific::GsBase; use x86_64::structures::tss::TaskStateSegment; use x86_64::VirtAddr; @@ -12,6 +14,8 @@ use x86_64::VirtAddr; use super::interrupts::{IrqStatistics, IRQ_COUNTERS}; use super::CPU_ONLINE; use crate::executor::task::AsyncTask; +#[cfg(feature = "smp")] +use crate::scheduler::SchedulerInput; use crate::scheduler::{CoreId, PerCoreScheduler}; #[repr(C)] @@ -29,6 +33,9 @@ pub(crate) struct CoreLocal { irq_statistics: &'static IrqStatistics, /// Queue of async tasks async_tasks: RefCell>, + /// Queues to handle incoming requests from the other cores + #[cfg(feature = "smp")] + pub scheduler_input: InterruptTicketMutex, } impl CoreLocal { @@ -52,6 +59,8 @@ impl CoreLocal { kernel_stack: Cell::new(0), irq_statistics, async_tasks: RefCell::new(Vec::new()), + #[cfg(feature = "smp")] + scheduler_input: InterruptTicketMutex::new(SchedulerInput::new()), }; let this = if core_id == 0 { take_static::take_static! { diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index e55032ce60..af687a65d4 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -35,7 +35,7 @@ static TASKS: InterruptTicketMutex> = pub type CoreId = u32; #[cfg(feature = "smp")] -struct SchedulerInput { +pub struct SchedulerInput { /// Queue of new tasks new_tasks: VecDeque, /// Queue of task, which are wakeup by another core @@ -74,9 +74,6 @@ pub struct PerCoreScheduler { finished_tasks: VecDeque>>, /// Queue of blocked tasks, sorted by wakeup time. blocked_tasks: BlockedTaskQueue, - /// Queues to handle incoming requests from the other cores - #[cfg(feature = "smp")] - input: InterruptTicketMutex, } struct NewTask { @@ -447,7 +444,7 @@ impl PerCoreScheduler { #[cfg(all(target_arch = "x86_64", feature = "smp"))] pub fn check_input(&mut self) { - let mut input_locked = self.input.lock(); + let mut input_locked = CoreLocal::get().scheduler_input.lock(); while let Some(task) = input_locked.wakeup_tasks.pop_front() { self.blocked_tasks.custom_wakeup(task); @@ -689,18 +686,16 @@ pub fn add_current_core() { ready_queue: PriorityTaskQueue::new(), finished_tasks: VecDeque::new(), blocked_tasks: BlockedTaskQueue::new(), - #[cfg(feature = "smp")] - input: InterruptTicketMutex::new(SchedulerInput::new()), }); let scheduler = Box::into_raw(boxed_scheduler); set_core_scheduler(scheduler); #[cfg(feature = "smp")] { - let scheduler = unsafe { scheduler.as_ref().unwrap() }; - SCHEDULER_INPUTS - .lock() - .insert(core_id.try_into().unwrap(), &scheduler.input); + SCHEDULER_INPUTS.lock().insert( + core_id.try_into().unwrap(), + &CoreLocal::get().scheduler_input, + ); } } From 4deb95dd14814de2260ba69bf86803195f93e28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Wed, 13 Sep 2023 14:11:50 +0200 Subject: [PATCH 3/7] fix(task): assert task found in custom_wakeup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/scheduler/task.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scheduler/task.rs b/src/scheduler/task.rs index caed8dd6ed..755fbf71da 100644 --- a/src/scheduler/task.rs +++ b/src/scheduler/task.rs @@ -609,12 +609,14 @@ impl BlockedTaskQueue { ); } - break; + return; } first_task = false; cursor.move_next(); } + + unreachable!(); } /// Wakes up all tasks whose wakeup time has elapsed. From a80bc914fee11e0152efab6837517efa580415a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Wed, 13 Sep 2023 22:29:52 +0200 Subject: [PATCH 4/7] fix(task): don't access scheduler from BlockedTaskQueue::wakeup_task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/scheduler/mod.rs | 17 ++++++++--- src/scheduler/task.rs | 65 ++++++++++++++++++++++++------------------- 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index af687a65d4..03eae19910 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -293,19 +293,27 @@ impl PerCoreScheduler { pub fn handle_waiting_tasks(&mut self) { without_interrupts(|| { crate::executor::run(); - self.blocked_tasks.handle_waiting_tasks(); + for task in self.blocked_tasks.handle_waiting_tasks() { + self.ready_queue.push(task); + } }); } #[cfg(not(feature = "smp"))] pub fn custom_wakeup(&mut self, task: TaskHandle) { - without_interrupts(|| self.blocked_tasks.custom_wakeup(task)); + without_interrupts(|| { + let task = self.blocked_tasks.custom_wakeup(task); + self.ready_queue.push(task); + }); } #[cfg(feature = "smp")] pub fn custom_wakeup(&mut self, task: TaskHandle) { if task.get_core_id() == self.core_id { - without_interrupts(|| self.blocked_tasks.custom_wakeup(task)); + without_interrupts(|| { + let task = self.blocked_tasks.custom_wakeup(task); + self.ready_queue.push(task); + }); } else { get_scheduler_input(task.get_core_id()) .lock() @@ -447,7 +455,8 @@ impl PerCoreScheduler { let mut input_locked = CoreLocal::get().scheduler_input.lock(); while let Some(task) = input_locked.wakeup_tasks.pop_front() { - self.blocked_tasks.custom_wakeup(task); + let task = self.blocked_tasks.custom_wakeup(task); + self.ready_queue.push(task); } while let Some(new_task) = input_locked.new_tasks.pop_front() { diff --git a/src/scheduler/task.rs b/src/scheduler/task.rs index 755fbf71da..7078e9a3f0 100644 --- a/src/scheduler/task.rs +++ b/src/scheduler/task.rs @@ -1,6 +1,7 @@ use alloc::boxed::Box; use alloc::collections::{LinkedList, VecDeque}; use alloc::rc::Rc; +use alloc::vec::Vec; use core::cell::RefCell; use core::cmp::Ordering; use core::fmt; @@ -463,31 +464,26 @@ impl BlockedTaskQueue { } fn wakeup_task(task: Rc>) { - { - let mut borrowed = task.borrow_mut(); - debug!( - "Waking up task {} on core {}", - borrowed.id, borrowed.core_id - ); - - assert!( - borrowed.core_id == core_id(), - "Try to wake up task {} on the wrong core {} != {}", - borrowed.id, - borrowed.core_id, - core_id() - ); + let mut borrowed = task.borrow_mut(); + debug!( + "Waking up task {} on core {}", + borrowed.id, borrowed.core_id + ); - assert!( - borrowed.status == TaskStatus::Blocked, - "Trying to wake up task {} which is not blocked", - borrowed.id - ); - borrowed.status = TaskStatus::Ready; - } + assert!( + borrowed.core_id == core_id(), + "Try to wake up task {} on the wrong core {} != {}", + borrowed.id, + borrowed.core_id, + core_id() + ); - // Add the task to the ready queue. - core_scheduler().ready_queue.push(task); + assert!( + borrowed.status == TaskStatus::Blocked, + "Trying to wake up task {} which is not blocked", + borrowed.id + ); + borrowed.status = TaskStatus::Ready; } #[cfg(any(feature = "tcp", feature = "udp"))] @@ -560,7 +556,7 @@ impl BlockedTaskQueue { } /// Manually wake up a blocked task. - pub fn custom_wakeup(&mut self, task: TaskHandle) { + pub fn custom_wakeup(&mut self, task: TaskHandle) -> Rc> { let mut first_task = true; let mut cursor = self.list.cursor_front_mut(); @@ -574,8 +570,8 @@ impl BlockedTaskQueue { // Loop through all blocked tasks to find it. while let Some(node) = cursor.current() { if node.task.borrow().id == task.get_id() { - // Remove it from the list of blocked tasks and wake it up. - Self::wakeup_task(node.task.clone()); + // Remove it from the list of blocked tasks. + let task_ref = node.task.clone(); cursor.remove_current(); // If this is the first task, adjust the One-Shot Timer to fire at the @@ -609,7 +605,10 @@ impl BlockedTaskQueue { ); } - return; + // Wake it up. + Self::wakeup_task(task_ref.clone()); + + return task_ref; } first_task = false; @@ -623,7 +622,7 @@ impl BlockedTaskQueue { /// /// Should be called by the One-Shot Timer interrupt handler when the wakeup time for /// at least one task has elapsed. - pub fn handle_waiting_tasks(&mut self) { + pub fn handle_waiting_tasks(&mut self) -> Vec>> { // Get the current time. let time = arch::processor::get_timer_ticks(); @@ -636,6 +635,8 @@ impl BlockedTaskQueue { } } + let mut tasks = vec![]; + // Loop through all blocked tasks. let mut cursor = self.list.cursor_front_mut(); while let Some(node) = cursor.current() { @@ -647,7 +648,7 @@ impl BlockedTaskQueue { } // Otherwise, this task has elapsed, so remove it from the list and wake it up. - Self::wakeup_task(node.task.clone()); + tasks.push(node.task.clone()); cursor.remove_current(); } @@ -675,5 +676,11 @@ impl BlockedTaskQueue { .current() .map_or_else(|| None, |node| node.wakeup_time), ); + + for task in tasks.iter().cloned() { + Self::wakeup_task(task); + } + + tasks } } From a434f0e72b92cdd3d8d582c8d0f75cd8643d3057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Mon, 18 Sep 2023 19:40:02 +0200 Subject: [PATCH 5/7] refactor(scheduler): move reschedule, add_network_timer, and exit to extension trait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/x86_64/kernel/apic.rs | 2 + src/arch/x86_64/kernel/scheduler.rs | 1 + src/drivers/net/mod.rs | 2 + src/executor/network.rs | 1 + src/lib.rs | 1 + src/scheduler/mod.rs | 200 +++++++++++++++------------- src/synch/futex.rs | 1 + src/synch/recmutex.rs | 1 + src/synch/semaphore.rs | 1 + src/syscalls/tasks.rs | 1 + 10 files changed, 117 insertions(+), 94 deletions(-) diff --git a/src/arch/x86_64/kernel/apic.rs b/src/arch/x86_64/kernel/apic.rs index fc8804b25a..6f4850f3b8 100644 --- a/src/arch/x86_64/kernel/apic.rs +++ b/src/arch/x86_64/kernel/apic.rs @@ -225,6 +225,8 @@ extern "x86-interrupt" fn spurious_interrupt_handler(stack_frame: interrupts::Ex #[cfg(feature = "smp")] extern "x86-interrupt" fn wakeup_handler(_stack_frame: interrupts::ExceptionStackFrame) { + use crate::scheduler::PerCoreSchedulerExt; + debug!("Received Wakeup Interrupt"); increment_irq_counter(WAKEUP_INTERRUPT_NUMBER); let core_scheduler = core_scheduler(); diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 41d11e3954..10fb48708b 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -16,6 +16,7 @@ use crate::arch::x86_64::mm::{PhysAddr, VirtAddr}; use crate::config::*; use crate::kernel; use crate::scheduler::task::{Task, TaskFrame}; +use crate::scheduler::PerCoreSchedulerExt; #[repr(C, packed)] struct State { diff --git a/src/drivers/net/mod.rs b/src/drivers/net/mod.rs index 64fbe47305..1144912e99 100644 --- a/src/drivers/net/mod.rs +++ b/src/drivers/net/mod.rs @@ -70,6 +70,8 @@ pub(crate) fn network_irqhandler(_state: &State) -> bool { #[cfg(target_arch = "x86_64")] pub(crate) extern "x86-interrupt" fn network_irqhandler(_stack_frame: ExceptionStackFrame) { + use crate::scheduler::PerCoreSchedulerExt; + debug!("Receive network interrupt"); apic::eoi(); let _ = _irqhandler(); diff --git a/src/executor/network.rs b/src/executor/network.rs index 22d98ca2ff..78122626e5 100644 --- a/src/executor/network.rs +++ b/src/executor/network.rs @@ -29,6 +29,7 @@ use crate::drivers::net::NetworkDriver; use crate::drivers::pci::get_network_driver; use crate::executor::device::HermitNet; use crate::executor::{spawn, TaskNotify}; +use crate::scheduler::PerCoreSchedulerExt; pub(crate) enum NetworkState<'a> { Missing, diff --git a/src/lib.rs b/src/lib.rs index 29cb0edf2b..a9c1d34b30 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ use mm::allocator::LockedAllocator; pub(crate) use crate::arch::*; pub(crate) use crate::config::*; use crate::kernel::is_uhyve_with_pci; +use crate::scheduler::PerCoreSchedulerExt; pub use crate::syscalls::*; #[macro_use] diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 03eae19910..26dc21b1d4 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -76,6 +76,112 @@ pub struct PerCoreScheduler { blocked_tasks: BlockedTaskQueue, } +pub trait PerCoreSchedulerExt { + /// Triggers the scheduler to reschedule the tasks. + /// Interrupt flag will be cleared during the reschedule + fn reschedule(self); + + #[cfg(any(feature = "tcp", feature = "udp"))] + fn add_network_timer(self, wakeup_time: Option); + + /// Terminate the current task on the current core. + fn exit(self, exit_code: i32) -> !; +} + +impl PerCoreSchedulerExt for &mut PerCoreScheduler { + #[cfg(target_arch = "x86_64")] + fn reschedule(self) { + without_interrupts(|| { + if let Some(last_stack_pointer) = self.scheduler() { + let (new_stack_pointer, is_idle) = { + let borrowed = self.current_task.borrow(); + ( + borrowed.last_stack_pointer, + borrowed.status == TaskStatus::Idle, + ) + }; + + if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { + unsafe { + switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); + } + } else { + unsafe { + switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); + } + } + } + }) + } + + /// Trigger an interrupt to reschedule the system + #[cfg(target_arch = "aarch64")] + fn reschedule(self) { + use core::arch::asm; + + use arm_gic::gicv3::{GicV3, IntId, SgiTarget}; + + use crate::interrupts::SGI_RESCHED; + + unsafe { + asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); + } + + let reschedid = IntId::sgi(SGI_RESCHED.into()); + GicV3::send_sgi( + reschedid, + SgiTarget::List { + affinity3: 0, + affinity2: 0, + affinity1: 0, + target_list: 0b1, + }, + ); + + interrupts::enable(); + } + + #[cfg(any(feature = "tcp", feature = "udp"))] + fn add_network_timer(self, wakeup_time: Option) { + without_interrupts(|| { + self.blocked_tasks.add_network_timer(wakeup_time); + }) + } + + fn exit(self, exit_code: i32) -> ! { + without_interrupts(|| { + // Get the current task. + let mut current_task_borrowed = self.current_task.borrow_mut(); + assert_ne!( + current_task_borrowed.status, + TaskStatus::Idle, + "Trying to terminate the idle task" + ); + + // Finish the task and reschedule. + debug!( + "Finishing task {} with exit code {}", + current_task_borrowed.id, exit_code + ); + current_task_borrowed.status = TaskStatus::Finished; + NO_TASKS.fetch_sub(1, Ordering::SeqCst); + + let current_id = current_task_borrowed.id; + drop(current_task_borrowed); + + // wakeup tasks, which are waiting for task with the identifier id + if let Some(mut queue) = WAITING_TASKS.lock().remove(¤t_id) { + while let Some(task) = queue.pop_front() { + self.custom_wakeup(task); + } + } + }); + + self.reschedule(); + unreachable!() + } +} + struct NewTask { tid: TaskId, func: extern "C" fn(usize), @@ -169,40 +275,6 @@ impl PerCoreScheduler { tid } - /// Terminate the current task on the current core. - pub fn exit(&mut self, exit_code: i32) -> ! { - without_interrupts(|| { - // Get the current task. - let mut current_task_borrowed = self.current_task.borrow_mut(); - assert_ne!( - current_task_borrowed.status, - TaskStatus::Idle, - "Trying to terminate the idle task" - ); - - // Finish the task and reschedule. - debug!( - "Finishing task {} with exit code {}", - current_task_borrowed.id, exit_code - ); - current_task_borrowed.status = TaskStatus::Finished; - NO_TASKS.fetch_sub(1, Ordering::SeqCst); - - let current_id = current_task_borrowed.id; - drop(current_task_borrowed); - - // wakeup tasks, which are waiting for task with the identifier id - if let Some(mut queue) = WAITING_TASKS.lock().remove(¤t_id) { - while let Some(task) = queue.pop_front() { - self.custom_wakeup(task); - } - } - }); - - self.reschedule(); - unreachable!() - } - #[cfg(feature = "newlib")] fn clone_impl(&self, func: extern "C" fn(usize), arg: usize) -> TaskId { static NEXT_CORE_ID: AtomicU32 = AtomicU32::new(1); @@ -332,12 +404,6 @@ impl PerCoreScheduler { }); } - #[cfg(any(feature = "tcp", feature = "udp"))] - #[inline] - pub fn add_network_timer(&mut self, wakeup_time: Option) { - without_interrupts(|| self.blocked_tasks.add_network_timer(wakeup_time)) - } - #[inline] pub fn get_current_task_handle(&self) -> TaskHandle { without_interrupts(|| { @@ -465,60 +531,6 @@ impl PerCoreScheduler { } } - /// Triggers the scheduler to reschedule the tasks. - /// Interrupt flag will be cleared during the reschedule - #[cfg(target_arch = "x86_64")] - pub fn reschedule(&mut self) { - without_interrupts(|| { - if let Some(last_stack_pointer) = self.scheduler() { - let (new_stack_pointer, is_idle) = { - let borrowed = self.current_task.borrow(); - ( - borrowed.last_stack_pointer, - borrowed.status == TaskStatus::Idle, - ) - }; - - if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { - unsafe { - switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); - } - } else { - unsafe { - switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); - } - } - } - }) - } - - /// Trigger an interrupt to reschedule the system - #[cfg(target_arch = "aarch64")] - pub fn reschedule(&self) { - use core::arch::asm; - - use arm_gic::gicv3::{GicV3, IntId, SgiTarget}; - - use crate::interrupts::SGI_RESCHED; - - unsafe { - asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); - } - - let reschedid = IntId::sgi(SGI_RESCHED.into()); - GicV3::send_sgi( - reschedid, - SgiTarget::List { - affinity3: 0, - affinity2: 0, - affinity1: 0, - target_list: 0b1, - }, - ); - - interrupts::enable(); - } - /// Only the idle task should call this function. /// Set the idle task to halt state if not another /// available. diff --git a/src/synch/futex.rs b/src/synch/futex.rs index db1196d281..ded2428357 100644 --- a/src/synch/futex.rs +++ b/src/synch/futex.rs @@ -10,6 +10,7 @@ use crate::arch::kernel::core_local::core_scheduler; use crate::arch::kernel::processor::get_timer_ticks; use crate::errno::{EAGAIN, EINVAL, ETIMEDOUT}; use crate::scheduler::task::TaskHandlePriorityQueue; +use crate::scheduler::PerCoreSchedulerExt; // TODO: Replace with a concurrent hashmap. static PARKING_LOT: InterruptTicketMutex> = diff --git a/src/synch/recmutex.rs b/src/synch/recmutex.rs index 885fa9fdd1..9491124e84 100644 --- a/src/synch/recmutex.rs +++ b/src/synch/recmutex.rs @@ -2,6 +2,7 @@ use hermit_sync::TicketMutex; use crate::arch::core_local::*; use crate::scheduler::task::{TaskHandlePriorityQueue, TaskId}; +use crate::scheduler::PerCoreSchedulerExt; struct RecursiveMutexState { current_tid: Option, diff --git a/src/synch/semaphore.rs b/src/synch/semaphore.rs index 2bf81ec22a..1c508fb8a0 100644 --- a/src/synch/semaphore.rs +++ b/src/synch/semaphore.rs @@ -4,6 +4,7 @@ use hermit_sync::InterruptTicketMutex; use crate::arch::core_local::*; use crate::scheduler::task::TaskHandlePriorityQueue; +use crate::scheduler::PerCoreSchedulerExt; struct SemaphoreState { /// Resource available count diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index 8edbac4880..e4962dae15 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -14,6 +14,7 @@ use crate::errno::*; #[cfg(feature = "newlib")] use crate::mm::{task_heap_end, task_heap_start}; use crate::scheduler::task::{Priority, TaskHandle, TaskId}; +use crate::scheduler::PerCoreSchedulerExt; use crate::syscalls::timer::timespec; use crate::{arch, scheduler, syscalls}; From ab83ae12c1c860bf5a077742771160127d3853d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Mon, 18 Sep 2023 19:50:00 +0200 Subject: [PATCH 6/7] refactor(scheduler): make run take no arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/lib.rs | 8 +++----- src/scheduler/mod.rs | 9 +++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a9c1d34b30..0f87964021 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,7 +56,7 @@ use mm::allocator::LockedAllocator; pub(crate) use crate::arch::*; pub(crate) use crate::config::*; use crate::kernel::is_uhyve_with_pci; -use crate::scheduler::PerCoreSchedulerExt; +use crate::scheduler::{PerCoreScheduler, PerCoreSchedulerExt}; pub use crate::syscalls::*; #[macro_use] @@ -343,9 +343,8 @@ fn boot_processor_main() -> ! { // Start the initd task. scheduler::PerCoreScheduler::spawn(initd, 0, scheduler::task::NORMAL_PRIO, 0, USER_STACK_SIZE); - let core_scheduler = core_scheduler(); // Run the scheduler loop. - core_scheduler.run(); + PerCoreScheduler::run(); } /// Entry Point of HermitCore for an Application Processor @@ -359,9 +358,8 @@ fn application_processor_main() -> ! { synch_all_cores(); crate::executor::init(); - let core_scheduler = core_scheduler(); // Run the scheduler loop. - core_scheduler.run(); + PerCoreScheduler::run(); } #[cfg(target_os = "none")] diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 26dc21b1d4..b939d08e0e 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -534,19 +534,20 @@ impl PerCoreScheduler { /// Only the idle task should call this function. /// Set the idle task to halt state if not another /// available. - pub fn run(&mut self) -> ! { + pub fn run() -> ! { let backoff = Backoff::new(); loop { + let core_scheduler = core_scheduler(); interrupts::disable(); // run async tasks crate::executor::run(); // do housekeeping - self.cleanup_tasks(); + core_scheduler.cleanup_tasks(); - if self.ready_queue.is_empty() { + if core_scheduler.ready_queue.is_empty() { if backoff.is_completed() { interrupts::enable_and_wait(); } else { @@ -555,7 +556,7 @@ impl PerCoreScheduler { } } else { interrupts::enable(); - self.reschedule(); + core_scheduler.reschedule(); backoff.reset(); } } From 8cbf7f6610009d2f779883120143e98f824ff066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Mon, 18 Sep 2023 19:57:31 +0200 Subject: [PATCH 7/7] fix(core_local): put scheduler into RefCell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- src/arch/aarch64/kernel/core_local.rs | 17 ++++++++-------- src/arch/x86_64/kernel/apic.rs | 2 +- src/arch/x86_64/kernel/core_local.rs | 14 +++++++------ src/scheduler/mod.rs | 29 ++++++++++++++------------- src/synch/futex.rs | 18 +++++++++-------- src/synch/recmutex.rs | 4 ++-- src/synch/semaphore.rs | 2 +- src/syscalls/tasks.rs | 4 ++-- 8 files changed, 48 insertions(+), 42 deletions(-) diff --git a/src/arch/aarch64/kernel/core_local.rs b/src/arch/aarch64/kernel/core_local.rs index a700972923..44d4dffede 100644 --- a/src/arch/aarch64/kernel/core_local.rs +++ b/src/arch/aarch64/kernel/core_local.rs @@ -1,7 +1,7 @@ use alloc::boxed::Box; use alloc::vec::Vec; use core::arch::asm; -use core::cell::{Cell, RefCell, RefMut}; +use core::cell::{RefCell, RefMut}; use core::ptr; use core::sync::atomic::Ordering; @@ -20,7 +20,7 @@ pub(crate) struct CoreLocal { /// ID of the current Core. core_id: CoreId, /// Scheduler of the current Core. - scheduler: Cell<*mut PerCoreScheduler>, + scheduler: RefCell>, /// Interface to the interrupt counters irq_statistics: &'static IrqStatistics, /// Queue of async tasks @@ -44,7 +44,7 @@ impl CoreLocal { let this = Self { this: ptr::null_mut(), core_id, - scheduler: Cell::new(ptr::null_mut()), + scheduler: RefCell::new(None), irq_statistics, async_tasks: RefCell::new(Vec::new()), #[cfg(feature = "smp")] @@ -91,17 +91,18 @@ pub(crate) fn core_id() -> CoreId { } } -#[inline] -pub(crate) fn core_scheduler() -> &'static mut PerCoreScheduler { - unsafe { &mut *CoreLocal::get().scheduler.get() } +pub(crate) fn core_scheduler() -> RefMut<'static, PerCoreScheduler> { + RefMut::map(CoreLocal::get().scheduler.borrow_mut(), |scheduler| { + scheduler.as_mut().unwrap() + }) } pub(crate) fn async_tasks() -> RefMut<'static, Vec> { CoreLocal::get().async_tasks.borrow_mut() } -pub(crate) fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { - CoreLocal::get().scheduler.set(scheduler); +pub(crate) fn set_core_scheduler(scheduler: PerCoreScheduler) { + *CoreLocal::get().scheduler.borrow_mut() = Some(scheduler); } pub(crate) fn increment_irq_counter(irq_no: u8) { diff --git a/src/arch/x86_64/kernel/apic.rs b/src/arch/x86_64/kernel/apic.rs index 6f4850f3b8..1aab2ede7a 100644 --- a/src/arch/x86_64/kernel/apic.rs +++ b/src/arch/x86_64/kernel/apic.rs @@ -229,7 +229,7 @@ extern "x86-interrupt" fn wakeup_handler(_stack_frame: interrupts::ExceptionStac debug!("Received Wakeup Interrupt"); increment_irq_counter(WAKEUP_INTERRUPT_NUMBER); - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); core_scheduler.check_input(); eoi(); if core_scheduler.is_scheduling() { diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index bf6210f5fa..50509a9e22 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -24,7 +24,7 @@ pub(crate) struct CoreLocal { /// Sequential ID of this CPU Core. core_id: CoreId, /// Scheduler for this CPU Core. - scheduler: Cell<*mut PerCoreScheduler>, + scheduler: RefCell>, /// Task State Segment (TSS) allocated for this CPU Core. pub tss: Cell<*mut TaskStateSegment>, /// start address of the kernel stack @@ -54,7 +54,7 @@ impl CoreLocal { let this = Self { this: ptr::null_mut(), core_id, - scheduler: Cell::new(ptr::null_mut()), + scheduler: RefCell::new(None), tss: Cell::new(ptr::null_mut()), kernel_stack: Cell::new(0), irq_statistics, @@ -101,16 +101,18 @@ pub(crate) fn core_id() -> CoreId { } } -pub(crate) fn core_scheduler() -> &'static mut PerCoreScheduler { - unsafe { &mut *CoreLocal::get().scheduler.get() } +pub(crate) fn core_scheduler() -> RefMut<'static, PerCoreScheduler> { + RefMut::map(CoreLocal::get().scheduler.borrow_mut(), |scheduler| { + scheduler.as_mut().unwrap() + }) } pub(crate) fn async_tasks() -> RefMut<'static, Vec> { CoreLocal::get().async_tasks.borrow_mut() } -pub(crate) fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { - CoreLocal::get().scheduler.set(scheduler); +pub(crate) fn set_core_scheduler(scheduler: PerCoreScheduler) { + *CoreLocal::get().scheduler.borrow_mut() = Some(scheduler); } pub(crate) fn increment_irq_counter(irq_no: u8) { diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index b939d08e0e..53b75993e9 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -1,9 +1,8 @@ -use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; use alloc::rc::Rc; #[cfg(feature = "smp")] use alloc::vec::Vec; -use core::cell::RefCell; +use core::cell::{RefCell, RefMut}; use core::sync::atomic::{AtomicU32, Ordering}; use crossbeam_utils::Backoff; @@ -88,9 +87,9 @@ pub trait PerCoreSchedulerExt { fn exit(self, exit_code: i32) -> !; } -impl PerCoreSchedulerExt for &mut PerCoreScheduler { +impl PerCoreSchedulerExt for RefMut<'_, PerCoreScheduler> { #[cfg(target_arch = "x86_64")] - fn reschedule(self) { + fn reschedule(mut self) { without_interrupts(|| { if let Some(last_stack_pointer) = self.scheduler() { let (new_stack_pointer, is_idle) = { @@ -102,10 +101,12 @@ impl PerCoreSchedulerExt for &mut PerCoreScheduler { }; if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { + drop(self); unsafe { switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); } } else { + drop(self); unsafe { switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); } @@ -123,6 +124,8 @@ impl PerCoreSchedulerExt for &mut PerCoreScheduler { use crate::interrupts::SGI_RESCHED; + drop(self); + unsafe { asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); } @@ -142,13 +145,14 @@ impl PerCoreSchedulerExt for &mut PerCoreScheduler { } #[cfg(any(feature = "tcp", feature = "udp"))] - fn add_network_timer(self, wakeup_time: Option) { + fn add_network_timer(mut self, wakeup_time: Option) { without_interrupts(|| { self.blocked_tasks.add_network_timer(wakeup_time); + drop(self); }) } - fn exit(self, exit_code: i32) -> ! { + fn exit(mut self, exit_code: i32) -> ! { without_interrupts(|| { // Get the current task. let mut current_task_borrowed = self.current_task.borrow_mut(); @@ -538,7 +542,7 @@ impl PerCoreScheduler { let backoff = Backoff::new(); loop { - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); interrupts::disable(); // run async tasks @@ -548,6 +552,7 @@ impl PerCoreScheduler { core_scheduler.cleanup_tasks(); if core_scheduler.ready_queue.is_empty() { + drop(core_scheduler); if backoff.is_completed() { interrupts::enable_and_wait(); } else { @@ -698,7 +703,7 @@ pub fn add_current_core() { "Initializing scheduler for core {} with idle task {}", core_id, tid ); - let boxed_scheduler = Box::new(PerCoreScheduler { + set_core_scheduler(PerCoreScheduler { #[cfg(feature = "smp")] core_id, current_task: idle_task.clone(), @@ -709,9 +714,6 @@ pub fn add_current_core() { finished_tasks: VecDeque::new(), blocked_tasks: BlockedTaskQueue::new(), }); - - let scheduler = Box::into_raw(boxed_scheduler); - set_core_scheduler(scheduler); #[cfg(feature = "smp")] { SCHEDULER_INPUTS.lock().insert( @@ -728,11 +730,9 @@ fn get_scheduler_input(core_id: CoreId) -> &'static InterruptTicketMutex Result<(), ()> { - let core_scheduler = core_scheduler(); - debug!( "Task {} is waiting for task {}", - core_scheduler.get_current_task_id(), + core_scheduler().get_current_task_id(), id ); @@ -740,6 +740,7 @@ pub fn join(id: TaskId) -> Result<(), ()> { let mut waiting_tasks_guard = WAITING_TASKS.lock(); if let Some(queue) = waiting_tasks_guard.get_mut(&id) { + let mut core_scheduler = core_scheduler(); queue.push_back(core_scheduler.get_current_task_handle()); core_scheduler.block_current_task(None); diff --git a/src/synch/futex.rs b/src/synch/futex.rs index ded2428357..a22d5a4ebe 100644 --- a/src/synch/futex.rs +++ b/src/synch/futex.rs @@ -52,14 +52,15 @@ pub(crate) fn futex_wait( timeout }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); scheduler.block_current_task(wakeup_time); let handle = scheduler.get_current_task_handle(); parking_lot.entry(addr(address)).or_default().push(handle); drop(parking_lot); + drop(scheduler); loop { - scheduler.reschedule(); + core_scheduler().reschedule(); let mut parking_lot = PARKING_LOT.lock(); if matches!(wakeup_time, Some(t) if t <= get_timer_ticks()) { @@ -88,7 +89,7 @@ pub(crate) fn futex_wait( } else { // A spurious wakeup occurred, sleep again. // Tasks do not change core, so the handle in the parking lot is still current. - scheduler.block_current_task(wakeup_time); + core_scheduler().block_current_task(wakeup_time); } } drop(parking_lot); @@ -121,14 +122,15 @@ pub(crate) fn futex_wait_and_set( timeout }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); scheduler.block_current_task(wakeup_time); let handle = scheduler.get_current_task_handle(); parking_lot.entry(addr(address)).or_default().push(handle); drop(parking_lot); + drop(scheduler); loop { - scheduler.reschedule(); + core_scheduler().reschedule(); let mut parking_lot = PARKING_LOT.lock(); if matches!(wakeup_time, Some(t) if t <= get_timer_ticks()) { @@ -157,7 +159,7 @@ pub(crate) fn futex_wait_and_set( } else { // A spurious wakeup occurred, sleep again. // Tasks do not change core, so the handle in the parking lot is still current. - scheduler.block_current_task(wakeup_time); + core_scheduler().block_current_task(wakeup_time); } } drop(parking_lot); @@ -178,7 +180,7 @@ pub(crate) fn futex_wake(address: &AtomicU32, count: i32) -> i32 { Entry::Vacant(_) => return 0, }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); let mut woken = 0; while woken != count || count == i32::MAX { match queue.get_mut().pop() { @@ -213,7 +215,7 @@ pub(crate) fn futex_wake_or_set(address: &AtomicU32, count: i32, new_value: u32) } }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); let mut woken = 0; while woken != count || count == i32::MAX { match queue.get_mut().pop() { diff --git a/src/synch/recmutex.rs b/src/synch/recmutex.rs index 9491124e84..03520a97f5 100644 --- a/src/synch/recmutex.rs +++ b/src/synch/recmutex.rs @@ -27,10 +27,10 @@ impl RecursiveMutex { pub fn acquire(&self) { // Get information about the current task. - let core_scheduler = core_scheduler(); - let tid = core_scheduler.get_current_task_id(); + let tid = core_scheduler().get_current_task_id(); loop { + let mut core_scheduler = core_scheduler(); { let mut locked_state = self.state.lock(); diff --git a/src/synch/semaphore.rs b/src/synch/semaphore.rs index 1c508fb8a0..b0bb5558d9 100644 --- a/src/synch/semaphore.rs +++ b/src/synch/semaphore.rs @@ -68,12 +68,12 @@ impl Semaphore { pub fn acquire(&self, time: Option) -> bool { #[cfg(feature = "smp")] let backoff = Backoff::new(); - let core_scheduler = core_scheduler(); let wakeup_time = time.map(|ms| crate::arch::processor::get_timer_ticks() + ms * 1000); // Loop until we have acquired the semaphore. loop { + let mut core_scheduler = core_scheduler(); let mut locked_state = self.state.lock(); if locked_state.count > 0 { diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index e4962dae15..3edc6a8204 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -116,7 +116,7 @@ pub(crate) extern "C" fn __sys_usleep(usecs: u64) { // Enough time to set a wakeup timer and block the current task. debug!("sys_usleep blocking the task for {} microseconds", usecs); let wakeup_time = arch::processor::get_timer_ticks() + usecs; - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); core_scheduler.block_current_task(Some(wakeup_time)); // Switch to the next task. @@ -300,7 +300,7 @@ static BLOCKED_TASKS: InterruptTicketMutex> = extern "C" fn __sys_block_current_task(timeout: &Option) { let wakeup_time = timeout.map(|t| arch::processor::get_timer_ticks() + t * 1000); - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); let handle = core_scheduler.get_current_task_handle(); let tid = core_scheduler.get_current_task_id();