From 3a648bc0aec24fcd2f2d90150d81735b1e2cee12 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Thu, 3 Nov 2022 21:36:48 -0700 Subject: [PATCH 01/55] split sub app runner into 2 separate closures --- crates/bevy_app/src/app.rs | 11 +++++++---- crates/bevy_render/src/lib.rs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index d88a82059126c..c2f5a0b915d2b 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -88,7 +88,8 @@ impl Debug for App { /// Each `SubApp` has its own [`Schedule`] and [`World`], enabling a separation of concerns. struct SubApp { app: App, - runner: Box, + extract: Box, + runner: Box, } impl Debug for SubApp { @@ -150,8 +151,8 @@ impl App { self.schedule.run(&mut self.world); for sub_app in self.sub_apps.values_mut() { - (sub_app.runner)(&mut self.world, &mut sub_app.app); - sub_app.app.world.clear_trackers(); + (sub_app.extract)(&mut self.world, &mut sub_app.app); + (sub_app.runner)(&mut sub_app.app); } self.world.clear_trackers(); @@ -992,12 +993,14 @@ impl App { &mut self, label: impl AppLabel, app: App, - sub_app_runner: impl Fn(&mut World, &mut App) + 'static, + sub_app_extract: impl Fn(&mut World, &mut App) + 'static, + sub_app_runner: impl Fn(&mut App) + 'static, ) -> &mut Self { self.sub_apps.insert( label.as_label(), SubApp { app, + extract: Box::new(sub_app_extract), runner: Box::new(sub_app_runner), }, ); diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index f595023588e76..cd2fbe0cad957 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -245,7 +245,7 @@ impl Plugin for RenderPlugin { // extract extract(app_world, render_app); } - + }, |render_app| { { #[cfg(feature = "trace")] let _stage_span = From 115db67b31296064ba7a9e8e2e05c64120b0c70c Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Thu, 3 Nov 2022 22:32:31 -0700 Subject: [PATCH 02/55] bad pipelining --- crates/bevy_app/Cargo.toml | 1 + crates/bevy_app/src/app.rs | 34 +++++++++++++++++---------- crates/bevy_render/src/view/window.rs | 2 +- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/crates/bevy_app/Cargo.toml b/crates/bevy_app/Cargo.toml index 3705bde56eee5..a4800e8881117 100644 --- a/crates/bevy_app/Cargo.toml +++ b/crates/bevy_app/Cargo.toml @@ -20,6 +20,7 @@ bevy_derive = { path = "../bevy_derive", version = "0.9.0" } bevy_ecs = { path = "../bevy_ecs", version = "0.9.0", default-features = false } bevy_reflect = { path = "../bevy_reflect", version = "0.9.0", optional = true } bevy_utils = { path = "../bevy_utils", version = "0.9.0" } +bevy_tasks = { path = "../bevy_tasks", version = "0.9.0" } # other serde = { version = "1.0", features = ["derive"], optional = true } diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index c2f5a0b915d2b..fd1bb1f21f230 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -8,8 +8,10 @@ use bevy_ecs::{ SystemStage, }, system::Resource, + world::World, }; +use bevy_tasks::ComputeTaskPool; use bevy_utils::{tracing::debug, HashMap, HashSet}; use std::fmt::Debug; @@ -67,12 +69,13 @@ pub struct App { /// the application's event loop and advancing the [`Schedule`]. /// Typically, it is not configured manually, but set by one of Bevy's built-in plugins. /// See `bevy::winit::WinitPlugin` and [`ScheduleRunnerPlugin`](crate::schedule_runner::ScheduleRunnerPlugin). - pub runner: Box, + pub runner: Box, // send/sync bound is only required to make App Send/Sync /// A container of [`Stage`]s set to be run in a linear order. pub schedule: Schedule, sub_apps: HashMap, plugin_registry: Vec>, plugin_name_added: HashSet, + run_once: bool, } impl Debug for App { @@ -88,8 +91,8 @@ impl Debug for App { /// Each `SubApp` has its own [`Schedule`] and [`World`], enabling a separation of concerns. struct SubApp { app: App, - extract: Box, - runner: Box, + extract: Box, // Send + Sync bound is only required to make SubApp send sync + runner: Box, // this send sync bound is required since we're actually sending this function to another thread } impl Debug for SubApp { @@ -137,6 +140,7 @@ impl App { sub_apps: HashMap::default(), plugin_registry: Vec::default(), plugin_name_added: Default::default(), + run_once: false, } } @@ -148,12 +152,18 @@ impl App { pub fn update(&mut self) { #[cfg(feature = "trace")] let _bevy_frame_update_span = info_span!("frame").entered(); - self.schedule.run(&mut self.world); - - for sub_app in self.sub_apps.values_mut() { - (sub_app.extract)(&mut self.world, &mut sub_app.app); - (sub_app.runner)(&mut sub_app.app); - } + ComputeTaskPool::get().scope(|scope| { + if self.run_once { + for sub_app in self.sub_apps.values_mut() { + (sub_app.extract)(&mut self.world, &mut sub_app.app); + } + for sub_app in self.sub_apps.values_mut() { + scope.spawn(async { (sub_app.runner)(&mut sub_app.app) }); + } + } + self.schedule.run(&mut self.world); + }); + self.run_once = true; self.world.clear_trackers(); } @@ -802,7 +812,7 @@ impl App { /// App::new() /// .set_runner(my_runner); /// ``` - pub fn set_runner(&mut self, run_fn: impl Fn(App) + 'static) -> &mut Self { + pub fn set_runner(&mut self, run_fn: impl Fn(App) + 'static + Send + Sync) -> &mut Self { self.runner = Box::new(run_fn); self } @@ -993,8 +1003,8 @@ impl App { &mut self, label: impl AppLabel, app: App, - sub_app_extract: impl Fn(&mut World, &mut App) + 'static, - sub_app_runner: impl Fn(&mut App) + 'static, + sub_app_extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, + sub_app_runner: impl Fn(&mut App) + 'static + Send + Sync, ) -> &mut Self { self.sub_apps.insert( label.as_label(), diff --git a/crates/bevy_render/src/view/window.rs b/crates/bevy_render/src/view/window.rs index 8e6f82c66c911..256b5ad4dd29c 100644 --- a/crates/bevy_render/src/view/window.rs +++ b/crates/bevy_render/src/view/window.rs @@ -166,7 +166,7 @@ pub struct WindowSurfaces { pub fn prepare_windows( // By accessing a NonSend resource, we tell the scheduler to put this system on the main thread, // which is necessary for some OS s - _marker: NonSend, + // _marker: NonSend, // TODO: reenable marker as this will break some OS's mut windows: ResMut, mut window_surfaces: ResMut, render_device: Res, From 833e47c24b1144b40d89eea2f4e1e4e985fce63d Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Fri, 4 Nov 2022 22:53:30 -0700 Subject: [PATCH 03/55] fix issue with systems not running sometimes --- crates/bevy_tasks/src/task_pool.rs | 33 +++++++++++++----------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 099f96e93d006..8c843e04d73ce 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -267,29 +267,24 @@ impl TaskPool { if spawned.is_empty() { Vec::new() } else { - let get_results = async { - let mut results = Vec::with_capacity(spawned_ref.len()); - while let Ok(task) = spawned_ref.pop() { - results.push(task.await.unwrap()); - } - - results - }; + future::block_on(async move { + let get_results = async { + let mut results = Vec::with_capacity(scope.spawned.len()); + while let Ok(task) = spawned_ref.pop() { + results.push(task.await); + } - // Pin the futures on the stack. - pin!(get_results); + results + }; - loop { - if let Some(result) = future::block_on(future::poll_once(&mut get_results)) { - break result; + let tick_forever = async move { + loop { + task_scope_executor.tick().await; + } }; - std::panic::catch_unwind(|| { - executor.try_tick(); - task_scope_executor.try_tick(); - }) - .ok(); - } + executor.run(tick_forever).or(get_results).await + }) } } From 1bd49088fa6c124017254a865f9725e545d24863 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Sat, 5 Nov 2022 22:26:22 -0800 Subject: [PATCH 04/55] create a global main thread executor --- crates/bevy_core/src/task_pool_options.rs | 8 +++++++- crates/bevy_tasks/Cargo.toml | 1 + crates/bevy_tasks/src/lib.rs | 2 +- crates/bevy_tasks/src/task_pool.rs | 10 +++++++++- crates/bevy_tasks/src/usages.rs | 24 +++++++++++++++++++++++ 5 files changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/bevy_core/src/task_pool_options.rs b/crates/bevy_core/src/task_pool_options.rs index 4537354a69c05..942324e5cc72f 100644 --- a/crates/bevy_core/src/task_pool_options.rs +++ b/crates/bevy_core/src/task_pool_options.rs @@ -1,5 +1,7 @@ use bevy_ecs::prelude::Resource; -use bevy_tasks::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, TaskPoolBuilder}; +use bevy_tasks::{ + AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, MainThreadExecutor, TaskPoolBuilder, +}; use bevy_utils::tracing::trace; /// Defines a simple way to determine how many threads to use given the number of remaining cores @@ -149,5 +151,9 @@ impl TaskPoolOptions { .build() }); } + + { + MainThreadExecutor::init(); + } } } diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index e39e182dffedc..291456b5d02b5 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -15,6 +15,7 @@ async-channel = "1.4.2" async-task = "4.2.0" once_cell = "1.7" concurrent-queue = "2.0.0" +is_main_thread = "0.1.0" [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = "0.4" diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 802f6c267b7cf..3382a8b252996 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -20,7 +20,7 @@ pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; mod usages; #[cfg(not(target_arch = "wasm32"))] pub use usages::tick_global_task_pools_on_main_thread; -pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; +pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, MainThreadExecutor}; mod iter; pub use iter::ParallelIterator; diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 8c843e04d73ce..93da24b3f0adc 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -8,8 +8,10 @@ use std::{ use async_task::FallibleTask; use concurrent_queue::ConcurrentQueue; -use futures_lite::{future, pin, FutureExt}; +use futures_lite::{future, FutureExt}; +use is_main_thread::is_main_thread; +use crate::MainThreadExecutor; use crate::Task; /// Used to create a [`TaskPool`] @@ -279,6 +281,12 @@ impl TaskPool { let tick_forever = async move { loop { + if let Some(is_main) = is_main_thread() { + if is_main { + MainThreadExecutor::get().tick().await; + } + } + task_scope_executor.tick().await; } }; diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index 1d0c83b271c2f..e9f22af3fbb7b 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -17,6 +17,7 @@ use std::ops::Deref; static COMPUTE_TASK_POOL: OnceCell = OnceCell::new(); static ASYNC_COMPUTE_TASK_POOL: OnceCell = OnceCell::new(); static IO_TASK_POOL: OnceCell = OnceCell::new(); +static MAIN_THREAD_EXECUTOR: OnceCell = OnceCell::new(); /// A newtype for a task pool for CPU-intensive work that must be completed to deliver the next /// frame @@ -110,6 +111,29 @@ impl Deref for IoTaskPool { } } +pub struct MainThreadExecutor(async_executor::Executor<'static>); + +impl MainThreadExecutor { + pub fn init() -> &'static Self { + MAIN_THREAD_EXECUTOR.get_or_init(|| Self(async_executor::Executor::new())) + } + + pub fn get() -> &'static Self { + MAIN_THREAD_EXECUTOR.get().expect( + "A MainThreadExecutor has not been initialize yet. Please call \ + MainThreadExecutor::init beforehand", + ) + } +} + +impl Deref for MainThreadExecutor { + type Target = async_executor::Executor<'static>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + /// Used by `bevy_core` to tick the global tasks pools on the main thread. /// This will run a maximum of 100 local tasks per executor per call to this function. #[cfg(not(target_arch = "wasm32"))] From 33f8207a1cb676b0782580bbb942b7b41797554c Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Sun, 6 Nov 2022 09:31:26 -0800 Subject: [PATCH 05/55] replace task scope executor with main thread executor --- crates/bevy_tasks/src/task_pool.rs | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 93da24b3f0adc..37399333e8283 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -247,7 +247,7 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. let executor: &async_executor::Executor = &self.executor; let executor: &'env async_executor::Executor = unsafe { mem::transmute(executor) }; - let task_scope_executor = &async_executor::Executor::default(); + let task_scope_executor = MainThreadExecutor::init(); let task_scope_executor: &'env async_executor::Executor = unsafe { mem::transmute(task_scope_executor) }; let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); @@ -279,19 +279,27 @@ impl TaskPool { results }; - let tick_forever = async move { - loop { - if let Some(is_main) = is_main_thread() { - if is_main { - MainThreadExecutor::get().tick().await; + let is_main = if let Some(is_main) = is_main_thread() { + is_main + } else { + false + }; + + if is_main { + let tick_forever = async move { + loop { + if let Some(is_main) = is_main_thread() { + if is_main { + task_scope_executor.tick().await; + } } } + }; - task_scope_executor.tick().await; - } - }; - - executor.run(tick_forever).or(get_results).await + executor.run(tick_forever).or(get_results).await + } else { + executor.run(get_results).await + } }) } } From 1abddb88bbcce4106db4413855714a10a2b45a37 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Sun, 6 Nov 2022 09:43:29 -0800 Subject: [PATCH 06/55] reenable NonSendMarker on prepare_windows --- crates/bevy_render/src/view/window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_render/src/view/window.rs b/crates/bevy_render/src/view/window.rs index 256b5ad4dd29c..8e6f82c66c911 100644 --- a/crates/bevy_render/src/view/window.rs +++ b/crates/bevy_render/src/view/window.rs @@ -166,7 +166,7 @@ pub struct WindowSurfaces { pub fn prepare_windows( // By accessing a NonSend resource, we tell the scheduler to put this system on the main thread, // which is necessary for some OS s - // _marker: NonSend, // TODO: reenable marker as this will break some OS's + _marker: NonSend, mut windows: ResMut, mut window_surfaces: ResMut, render_device: Res, From ade5b77c5dc910c32c5171a1dc23bf50f5317deb Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Sun, 6 Nov 2022 10:59:36 -0800 Subject: [PATCH 07/55] make a safer abstraction for the main thread executor --- .../src/schedule/executor_parallel.rs | 4 +- crates/bevy_tasks/src/lib.rs | 5 +- crates/bevy_tasks/src/main_thread_executor.rs | 75 +++++++++++++++++++ crates/bevy_tasks/src/task_pool.rs | 39 ++++------ crates/bevy_tasks/src/usages.rs | 24 ------ 5 files changed, 96 insertions(+), 51 deletions(-) create mode 100644 crates/bevy_tasks/src/main_thread_executor.rs diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index 68dd1f1ea798d..8883b2922a386 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -236,7 +236,7 @@ impl ParallelExecutor { if system_data.is_send { scope.spawn(task); } else { - scope.spawn_on_scope(task); + scope.spawn_on_main(task); } #[cfg(test)] @@ -271,7 +271,7 @@ impl ParallelExecutor { if system_data.is_send { scope.spawn(task); } else { - scope.spawn_on_scope(task); + scope.spawn_on_main(task); } } } diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 3382a8b252996..3be4008fb86e9 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -20,7 +20,10 @@ pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; mod usages; #[cfg(not(target_arch = "wasm32"))] pub use usages::tick_global_task_pools_on_main_thread; -pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, MainThreadExecutor}; +pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; + +mod main_thread_executor; +pub use main_thread_executor::MainThreadExecutor; mod iter; pub use iter::ParallelIterator; diff --git a/crates/bevy_tasks/src/main_thread_executor.rs b/crates/bevy_tasks/src/main_thread_executor.rs new file mode 100644 index 0000000000000..b90a8abe03ea9 --- /dev/null +++ b/crates/bevy_tasks/src/main_thread_executor.rs @@ -0,0 +1,75 @@ +use std::{marker::PhantomData, sync::Arc}; + +use async_executor::{Executor, Task}; +use futures_lite::Future; +use is_main_thread::is_main_thread; +use once_cell::sync::OnceCell; + +static MAIN_THREAD_EXECUTOR: OnceCell = OnceCell::new(); + +/// Use to access the global main thread executor. Be aware that the main thread executor +/// only makes progress when it is ticked. This normally happens in `[TaskPool::scope]`. +#[derive(Debug)] +pub struct MainThreadExecutor(Arc>); + +impl MainThreadExecutor { + /// Initializes the global `[MainThreadExecutor]` instance. + pub fn init() -> &'static Self { + MAIN_THREAD_EXECUTOR.get_or_init(|| Self(Arc::new(Executor::new()))) + } + + /// Gets the global [`MainThreadExecutor`] instance. + /// + /// # Panics + /// Panics if no executor has been initialized yet. + pub fn get() -> &'static Self { + MAIN_THREAD_EXECUTOR.get().expect( + "A MainThreadExecutor has not been initialize yet. Please call \ + MainThreadExecutor::init beforehand", + ) + } + + /// Gets the `[MainThreadSpawner]` for the global main thread executor. + /// Use this to spawn tasks on the main thread. + pub fn spawner(&self) -> MainThreadSpawner<'static> { + MainThreadSpawner(self.0.clone()) + } + + /// Gets the `[MainThreadTicker]` for the global main thread executor. + /// Use this to tick the main thread executor. + /// Returns None if called on not the main thread. + pub fn ticker(&self) -> Option { + if let Some(is_main) = is_main_thread() { + if is_main { + return Some(MainThreadTicker { + executor: self.0.clone(), + _marker: PhantomData::default(), + }); + } + } + None + } +} + +#[derive(Debug)] +pub struct MainThreadSpawner<'a>(Arc>); +impl<'a> MainThreadSpawner<'a> { + /// Spawn a task on the main thread + pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { + self.0.spawn(future) + } +} + +#[derive(Debug)] +pub struct MainThreadTicker { + executor: Arc>, + // make type not send or sync + _marker: PhantomData<*const ()>, +} +impl MainThreadTicker { + /// Tick the main thread executor. + /// This needs to be called manually on the main thread if a `[TaskPool::scope]` is not active + pub fn tick<'a>(&'a self) -> impl Future + 'a { + self.executor.tick() + } +} diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 37399333e8283..674fb0169afc0 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -9,10 +9,9 @@ use std::{ use async_task::FallibleTask; use concurrent_queue::ConcurrentQueue; use futures_lite::{future, FutureExt}; -use is_main_thread::is_main_thread; -use crate::MainThreadExecutor; use crate::Task; +use crate::{main_thread_executor::MainThreadSpawner, MainThreadExecutor}; /// Used to create a [`TaskPool`] #[derive(Debug, Default, Clone)] @@ -247,16 +246,16 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. let executor: &async_executor::Executor = &self.executor; let executor: &'env async_executor::Executor = unsafe { mem::transmute(executor) }; - let task_scope_executor = MainThreadExecutor::init(); - let task_scope_executor: &'env async_executor::Executor = - unsafe { mem::transmute(task_scope_executor) }; + let main_thread_spawner = MainThreadExecutor::init().spawner(); + let main_thread_spawner: MainThreadSpawner<'env> = + unsafe { mem::transmute(main_thread_spawner) }; let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); let spawned_ref: &'env ConcurrentQueue> = unsafe { mem::transmute(&spawned) }; let scope = Scope { executor, - task_scope_executor, + main_thread_spawner, spawned: spawned_ref, scope: PhantomData, env: PhantomData, @@ -279,20 +278,10 @@ impl TaskPool { results }; - let is_main = if let Some(is_main) = is_main_thread() { - is_main - } else { - false - }; - - if is_main { + if let Some(main_thread_ticker) = MainThreadExecutor::get().ticker() { let tick_forever = async move { loop { - if let Some(is_main) = is_main_thread() { - if is_main { - task_scope_executor.tick().await; - } - } + main_thread_ticker.tick().await; } }; @@ -373,7 +362,7 @@ impl Drop for TaskPool { #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope async_executor::Executor<'scope>, - task_scope_executor: &'scope async_executor::Executor<'scope>, + main_thread_spawner: MainThreadSpawner<'scope>, spawned: &'scope ConcurrentQueue>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -402,8 +391,10 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// [`Scope::spawn`] instead, unless the provided future needs to run on the scope's thread. /// /// For more information, see [`TaskPool::scope`]. - pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { - let task = self.task_scope_executor.spawn(f).fallible(); + pub fn spawn_on_main + 'scope + Send>(&self, f: Fut) { + let main_thread_spawner: &MainThreadSpawner<'scope> = + unsafe { mem::transmute(&self.main_thread_spawner) }; + let task = main_thread_spawner.spawn(f).fallible(); // ConcurrentQueue only errors when closed or full, but we never // close and use an unbouded queue, so it is safe to unwrap self.spawned.push(task).unwrap(); @@ -487,7 +478,7 @@ mod tests { }); } else { let count_clone = local_count.clone(); - scope.spawn_on_scope(async move { + scope.spawn_on_main(async move { if *foo != 42 { panic!("not 42!?!?") } else { @@ -528,7 +519,7 @@ mod tests { }); let spawner = std::thread::current().id(); let inner_count_clone = count_clone.clone(); - scope.spawn_on_scope(async move { + scope.spawn_on_main(async move { inner_count_clone.fetch_add(1, Ordering::Release); if std::thread::current().id() != spawner { // NOTE: This check is using an atomic rather than simply panicing the @@ -603,7 +594,7 @@ mod tests { inner_count_clone.fetch_add(1, Ordering::Release); // spawning on the scope from another thread runs the futures on the scope's thread - scope.spawn_on_scope(async move { + scope.spawn_on_main(async move { inner_count_clone.fetch_add(1, Ordering::Release); if std::thread::current().id() != spawner { // NOTE: This check is using an atomic rather than simply panicing the diff --git a/crates/bevy_tasks/src/usages.rs b/crates/bevy_tasks/src/usages.rs index e9f22af3fbb7b..1d0c83b271c2f 100644 --- a/crates/bevy_tasks/src/usages.rs +++ b/crates/bevy_tasks/src/usages.rs @@ -17,7 +17,6 @@ use std::ops::Deref; static COMPUTE_TASK_POOL: OnceCell = OnceCell::new(); static ASYNC_COMPUTE_TASK_POOL: OnceCell = OnceCell::new(); static IO_TASK_POOL: OnceCell = OnceCell::new(); -static MAIN_THREAD_EXECUTOR: OnceCell = OnceCell::new(); /// A newtype for a task pool for CPU-intensive work that must be completed to deliver the next /// frame @@ -111,29 +110,6 @@ impl Deref for IoTaskPool { } } -pub struct MainThreadExecutor(async_executor::Executor<'static>); - -impl MainThreadExecutor { - pub fn init() -> &'static Self { - MAIN_THREAD_EXECUTOR.get_or_init(|| Self(async_executor::Executor::new())) - } - - pub fn get() -> &'static Self { - MAIN_THREAD_EXECUTOR.get().expect( - "A MainThreadExecutor has not been initialize yet. Please call \ - MainThreadExecutor::init beforehand", - ) - } -} - -impl Deref for MainThreadExecutor { - type Target = async_executor::Executor<'static>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - /// Used by `bevy_core` to tick the global tasks pools on the main thread. /// This will run a maximum of 100 local tasks per executor per call to this function. #[cfg(not(target_arch = "wasm32"))] From 1e0cecc181f6849266191af522f394ee5ab88295 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Sun, 6 Nov 2022 15:27:22 -0800 Subject: [PATCH 08/55] try to fix ci --still broken --- crates/bevy_app/src/app.rs | 5 ++--- crates/bevy_tasks/src/main_thread_executor.rs | 18 ++++++++++++++++-- crates/bevy_tasks/src/task_pool.rs | 15 ++++++++++++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index fd1bb1f21f230..90e211f3219d7 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -8,10 +8,9 @@ use bevy_ecs::{ SystemStage, }, system::Resource, - world::World, }; -use bevy_tasks::ComputeTaskPool; +use bevy_tasks::{ComputeTaskPool, TaskPool}; use bevy_utils::{tracing::debug, HashMap, HashSet}; use std::fmt::Debug; @@ -152,7 +151,7 @@ impl App { pub fn update(&mut self) { #[cfg(feature = "trace")] let _bevy_frame_update_span = info_span!("frame").entered(); - ComputeTaskPool::get().scope(|scope| { + ComputeTaskPool::init(TaskPool::default).scope(|scope| { if self.run_once { for sub_app in self.sub_apps.values_mut() { (sub_app.extract)(&mut self.world, &mut sub_app.app); diff --git a/crates/bevy_tasks/src/main_thread_executor.rs b/crates/bevy_tasks/src/main_thread_executor.rs index b90a8abe03ea9..8fc2acc7f9f87 100644 --- a/crates/bevy_tasks/src/main_thread_executor.rs +++ b/crates/bevy_tasks/src/main_thread_executor.rs @@ -10,7 +10,10 @@ static MAIN_THREAD_EXECUTOR: OnceCell = OnceCell::new(); /// Use to access the global main thread executor. Be aware that the main thread executor /// only makes progress when it is ticked. This normally happens in `[TaskPool::scope]`. #[derive(Debug)] -pub struct MainThreadExecutor(Arc>); +pub struct MainThreadExecutor( + // this is only pub crate for testing purposes, do not contruct otherwise + pub(crate) Arc>, +); impl MainThreadExecutor { /// Initializes the global `[MainThreadExecutor]` instance. @@ -39,6 +42,17 @@ impl MainThreadExecutor { /// Use this to tick the main thread executor. /// Returns None if called on not the main thread. pub fn ticker(&self) -> Option { + // always return ticker when testing to allow tests to run off main thread + dbg!("hjj"); + #[cfg(test)] + if true { + dbg!("blah"); + return Some(MainThreadTicker { + executor: self.0.clone(), + _marker: PhantomData::default(), + }); + } + if let Some(is_main) = is_main_thread() { if is_main { return Some(MainThreadTicker { @@ -69,7 +83,7 @@ pub struct MainThreadTicker { impl MainThreadTicker { /// Tick the main thread executor. /// This needs to be called manually on the main thread if a `[TaskPool::scope]` is not active - pub fn tick<'a>(&'a self) -> impl Future + 'a { + pub fn tick(&self) -> impl Future + '_ { self.executor.tick() } } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 674fb0169afc0..02126b8f3dd87 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -246,7 +246,15 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. let executor: &async_executor::Executor = &self.executor; let executor: &'env async_executor::Executor = unsafe { mem::transmute(executor) }; - let main_thread_spawner = MainThreadExecutor::init().spawner(); + + #[cfg(not(test))] + let main_thread_executor = MainThreadExecutor::init(); + // for testing configure a new instance of main thread executor for every scope + // this helps us pretend that the thread that an app or stage is constructed on is the main thread + #[cfg(test)] + let main_thread_executor = MainThreadExecutor(Arc::new(async_executor::Executor::new())); + + let main_thread_spawner = main_thread_executor.spawner(); let main_thread_spawner: MainThreadSpawner<'env> = unsafe { mem::transmute(main_thread_spawner) }; let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); @@ -278,9 +286,10 @@ impl TaskPool { results }; - if let Some(main_thread_ticker) = MainThreadExecutor::get().ticker() { + if let Some(main_thread_ticker) = main_thread_executor.ticker() { let tick_forever = async move { loop { + dbg!("tivk"); main_thread_ticker.tick().await; } }; @@ -455,7 +464,7 @@ mod tests { } #[test] - fn test_mixed_spawn_on_scope_and_spawn() { + fn test_mixed_spawn_on_main_and_spawn() { let pool = TaskPool::new(); let foo = Box::new(42); From 5498fe53072e32a99b337575db9aabc0ce64e127 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 7 Nov 2022 17:48:40 -0800 Subject: [PATCH 09/55] change scope to take a thread executor --- crates/bevy_app/src/app.rs | 15 +++-- crates/bevy_core/src/lib.rs | 2 + crates/bevy_core/src/task_pool_options.rs | 8 +-- crates/bevy_ecs/src/query/state.rs | 2 +- .../src/schedule/executor_parallel.rs | 32 +++++++-- crates/bevy_gltf/src/loader.rs | 2 +- crates/bevy_tasks/examples/busy_behavior.rs | 2 +- crates/bevy_tasks/examples/idle_behavior.rs | 2 +- crates/bevy_tasks/src/iter/mod.rs | 32 ++++----- crates/bevy_tasks/src/lib.rs | 2 +- crates/bevy_tasks/src/main_thread_executor.rs | 67 ++++++++----------- crates/bevy_tasks/src/slice.rs | 4 +- crates/bevy_tasks/src/task_pool.rs | 60 ++++++++--------- 13 files changed, 119 insertions(+), 111 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 90e211f3219d7..59fe7c8271ba3 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -4,8 +4,8 @@ use bevy_ecs::{ event::{Event, Events}, prelude::FromWorld, schedule::{ - IntoSystemDescriptor, Schedule, ShouldRun, Stage, StageLabel, State, StateData, SystemSet, - SystemStage, + IntoSystemDescriptor, MainThreadExecutor, Schedule, ShouldRun, Stage, StageLabel, State, + StateData, SystemSet, SystemStage, }, system::Resource, world::World, @@ -151,7 +151,11 @@ impl App { pub fn update(&mut self) { #[cfg(feature = "trace")] let _bevy_frame_update_span = info_span!("frame").entered(); - ComputeTaskPool::init(TaskPool::default).scope(|scope| { + let thread_executor = self + .world + .get_resource::() + .map(|e| e.0.clone()); + ComputeTaskPool::init(TaskPool::default).scope(thread_executor, |scope| { if self.run_once { for sub_app in self.sub_apps.values_mut() { (sub_app.extract)(&mut self.world, &mut sub_app.app); @@ -1001,10 +1005,13 @@ impl App { pub fn add_sub_app( &mut self, label: impl AppLabel, - app: App, + mut app: App, sub_app_extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, sub_app_runner: impl Fn(&mut App) + 'static + Send + Sync, ) -> &mut Self { + if let Some(executor) = self.world.get_resource::() { + app.world.insert_resource(executor.clone()); + } self.sub_apps.insert( label.as_label(), SubApp { diff --git a/crates/bevy_core/src/lib.rs b/crates/bevy_core/src/lib.rs index ee21feb2ec9ef..4b64f7efcc0e6 100644 --- a/crates/bevy_core/src/lib.rs +++ b/crates/bevy_core/src/lib.rs @@ -6,6 +6,7 @@ mod name; mod serde; mod task_pool_options; +use bevy_ecs::schedule::MainThreadExecutor; use bevy_ecs::system::{ResMut, Resource}; pub use bytemuck::{bytes_of, cast_slice, Pod, Zeroable}; pub use name::*; @@ -42,6 +43,7 @@ impl Plugin for CorePlugin { fn build(&self, app: &mut App) { // Setup the default bevy task pools self.task_pool_options.create_default_pools(); + app.insert_resource(MainThreadExecutor::new()); #[cfg(not(target_arch = "wasm32"))] app.add_system_to_stage( diff --git a/crates/bevy_core/src/task_pool_options.rs b/crates/bevy_core/src/task_pool_options.rs index 942324e5cc72f..4537354a69c05 100644 --- a/crates/bevy_core/src/task_pool_options.rs +++ b/crates/bevy_core/src/task_pool_options.rs @@ -1,7 +1,5 @@ use bevy_ecs::prelude::Resource; -use bevy_tasks::{ - AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, MainThreadExecutor, TaskPoolBuilder, -}; +use bevy_tasks::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool, TaskPoolBuilder}; use bevy_utils::tracing::trace; /// Defines a simple way to determine how many threads to use given the number of remaining cores @@ -151,9 +149,5 @@ impl TaskPoolOptions { .build() }); } - - { - MainThreadExecutor::init(); - } } } diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index b0f263e7fd948..a5d1f3c4f8ffc 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -990,7 +990,7 @@ impl QueryState { ) { // NOTE: If you are changing query iteration code, remember to update the following places, where relevant: // QueryIter, QueryIterationCursor, QueryManyIter, QueryCombinationIter, QueryState::for_each_unchecked_manual, QueryState::par_for_each_unchecked_manual - ComputeTaskPool::get().scope(|scope| { + ComputeTaskPool::get().scope(None, |scope| { if Q::IS_DENSE && F::IS_DENSE { let tables = &world.storages().tables; for table_id in &self.matched_table_ids { diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index 8883b2922a386..5a80aea31307a 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -1,11 +1,15 @@ +use std::sync::Arc; + +use crate as bevy_ecs; use crate::{ archetype::ArchetypeComponentId, query::Access, schedule::{ParallelSystemExecutor, SystemContainer}, + system::Resource, world::World, }; use async_channel::{Receiver, Sender}; -use bevy_tasks::{ComputeTaskPool, Scope, TaskPool}; +use bevy_tasks::{ComputeTaskPool, Scope, TaskPool, ThreadExecutor}; #[cfg(feature = "trace")] use bevy_utils::tracing::Instrument; use event_listener::Event; @@ -14,6 +18,22 @@ use fixedbitset::FixedBitSet; #[cfg(test)] use scheduling_event::*; +/// +#[derive(Resource, Default)] +pub struct MainThreadExecutor(pub Arc); + +impl MainThreadExecutor { + pub fn new() -> Self { + MainThreadExecutor(Arc::new(ThreadExecutor::new())) + } +} + +impl Clone for MainThreadExecutor { + fn clone(&self) -> Self { + MainThreadExecutor(self.0.clone()) + } +} + struct SystemSchedulingMetadata { /// Used to signal the system's task to start the system. start: Event, @@ -124,7 +144,11 @@ impl ParallelSystemExecutor for ParallelExecutor { } } - ComputeTaskPool::init(TaskPool::default).scope(|scope| { + let thread_executor = world + .get_resource::() + .map(|e| e.0.clone()); + + ComputeTaskPool::init(TaskPool::default).scope(thread_executor, |scope| { self.prepare_systems(scope, systems, world); if self.should_run.count_ones(..) == 0 { return; @@ -236,7 +260,7 @@ impl ParallelExecutor { if system_data.is_send { scope.spawn(task); } else { - scope.spawn_on_main(task); + scope.spawn_on_scope(task); } #[cfg(test)] @@ -271,7 +295,7 @@ impl ParallelExecutor { if system_data.is_send { scope.spawn(task); } else { - scope.spawn_on_main(task); + scope.spawn_on_scope(task); } } } diff --git a/crates/bevy_gltf/src/loader.rs b/crates/bevy_gltf/src/loader.rs index 79d19e2898ef4..8c992df2a47e1 100644 --- a/crates/bevy_gltf/src/loader.rs +++ b/crates/bevy_gltf/src/loader.rs @@ -409,7 +409,7 @@ async fn load_gltf<'a, 'b>( } else { #[cfg(not(target_arch = "wasm32"))] IoTaskPool::get() - .scope(|scope| { + .scope(None, |scope| { gltf.textures().for_each(|gltf_texture| { let linear_textures = &linear_textures; let load_context: &LoadContext = load_context; diff --git a/crates/bevy_tasks/examples/busy_behavior.rs b/crates/bevy_tasks/examples/busy_behavior.rs index 8a74034e0ca90..df78b2316251a 100644 --- a/crates/bevy_tasks/examples/busy_behavior.rs +++ b/crates/bevy_tasks/examples/busy_behavior.rs @@ -11,7 +11,7 @@ fn main() { .build(); let t0 = instant::Instant::now(); - pool.scope(|s| { + pool.scope(None, |s| { for i in 0..40 { s.spawn(async move { let now = instant::Instant::now(); diff --git a/crates/bevy_tasks/examples/idle_behavior.rs b/crates/bevy_tasks/examples/idle_behavior.rs index daa2eaf2e2a89..b1f5f2adb54b6 100644 --- a/crates/bevy_tasks/examples/idle_behavior.rs +++ b/crates/bevy_tasks/examples/idle_behavior.rs @@ -9,7 +9,7 @@ fn main() { .thread_name("Idle Behavior ThreadPool".to_string()) .build(); - pool.scope(|s| { + pool.scope(None, |s| { for i in 0..1 { s.spawn(async move { println!("Blocking for 10 seconds"); diff --git a/crates/bevy_tasks/src/iter/mod.rs b/crates/bevy_tasks/src/iter/mod.rs index 952bc53075551..2c3e2fa673138 100644 --- a/crates/bevy_tasks/src/iter/mod.rs +++ b/crates/bevy_tasks/src/iter/mod.rs @@ -34,7 +34,7 @@ where /// /// See [`Iterator::count()`](https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.count) fn count(mut self, pool: &TaskPool) -> usize { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.count() }); } @@ -105,7 +105,7 @@ where where F: FnMut(BatchIter::Item) + Send + Clone + Sync, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { @@ -195,7 +195,7 @@ where C: std::iter::FromIterator, BatchIter::Item: Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.collect::>() }); } @@ -216,7 +216,7 @@ where BatchIter::Item: Send + 'static, { let (mut a, mut b) = <(C, C)>::default(); - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.partition::, F>(newf) }); @@ -242,7 +242,7 @@ where F: FnMut(C, BatchIter::Item) -> C + Send + Sync + Clone, C: Clone + Send + Sync + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); let newi = init.clone(); @@ -260,7 +260,7 @@ where where F: FnMut(BatchIter::Item) -> bool + Send + Sync + Clone, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(mut batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.all(newf) }); @@ -279,7 +279,7 @@ where where F: FnMut(BatchIter::Item) -> bool + Send + Sync + Clone, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(mut batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.any(newf) }); @@ -299,7 +299,7 @@ where where F: FnMut(BatchIter::Item) -> bool + Send + Sync + Clone, { - let poses = pool.scope(|s| { + let poses = pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let mut newf = f.clone(); s.spawn(async move { @@ -332,7 +332,7 @@ where where BatchIter::Item: Ord + Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.max() }); } @@ -349,7 +349,7 @@ where where BatchIter::Item: Ord + Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.min() }); } @@ -368,7 +368,7 @@ where F: FnMut(&BatchIter::Item) -> R + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.max_by_key(newf) }); @@ -388,7 +388,7 @@ where F: FnMut(&BatchIter::Item, &BatchIter::Item) -> std::cmp::Ordering + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.max_by(newf) }); @@ -408,7 +408,7 @@ where F: FnMut(&BatchIter::Item) -> R + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.min_by_key(newf) }); @@ -428,7 +428,7 @@ where F: FnMut(&BatchIter::Item, &BatchIter::Item) -> std::cmp::Ordering + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.min_by(newf) }); @@ -482,7 +482,7 @@ where S: std::iter::Sum + Send + 'static, R: std::iter::Sum, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.sum() }); } @@ -499,7 +499,7 @@ where S: std::iter::Product + Send + 'static, R: std::iter::Product, { - pool.scope(|s| { + pool.scope(None, |s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.product() }); } diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index 3be4008fb86e9..ce35e8ee4b54f 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -23,7 +23,7 @@ pub use usages::tick_global_task_pools_on_main_thread; pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; mod main_thread_executor; -pub use main_thread_executor::MainThreadExecutor; +pub use main_thread_executor::ThreadExecutor; mod iter; pub use iter::ParallelIterator; diff --git a/crates/bevy_tasks/src/main_thread_executor.rs b/crates/bevy_tasks/src/main_thread_executor.rs index 8fc2acc7f9f87..2e39cc230b778 100644 --- a/crates/bevy_tasks/src/main_thread_executor.rs +++ b/crates/bevy_tasks/src/main_thread_executor.rs @@ -1,66 +1,53 @@ -use std::{marker::PhantomData, sync::Arc}; +use std::{ + marker::PhantomData, + sync::Arc, + thread::{self, ThreadId}, +}; use async_executor::{Executor, Task}; use futures_lite::Future; -use is_main_thread::is_main_thread; -use once_cell::sync::OnceCell; - -static MAIN_THREAD_EXECUTOR: OnceCell = OnceCell::new(); /// Use to access the global main thread executor. Be aware that the main thread executor /// only makes progress when it is ticked. This normally happens in `[TaskPool::scope]`. #[derive(Debug)] -pub struct MainThreadExecutor( +pub struct ThreadExecutor { // this is only pub crate for testing purposes, do not contruct otherwise - pub(crate) Arc>, -); + executor: Arc>, + thread_id: ThreadId, +} -impl MainThreadExecutor { - /// Initializes the global `[MainThreadExecutor]` instance. - pub fn init() -> &'static Self { - MAIN_THREAD_EXECUTOR.get_or_init(|| Self(Arc::new(Executor::new()))) +impl Default for ThreadExecutor { + fn default() -> Self { + Self { + executor: Arc::new(Executor::new()), + thread_id: thread::current().id(), + } } +} - /// Gets the global [`MainThreadExecutor`] instance. - /// - /// # Panics - /// Panics if no executor has been initialized yet. - pub fn get() -> &'static Self { - MAIN_THREAD_EXECUTOR.get().expect( - "A MainThreadExecutor has not been initialize yet. Please call \ - MainThreadExecutor::init beforehand", - ) +impl ThreadExecutor { + /// Initializes the global `[MainThreadExecutor]` instance. + pub fn new() -> Self { + Self::default() } /// Gets the `[MainThreadSpawner]` for the global main thread executor. /// Use this to spawn tasks on the main thread. pub fn spawner(&self) -> MainThreadSpawner<'static> { - MainThreadSpawner(self.0.clone()) + MainThreadSpawner(self.executor.clone()) } - /// Gets the `[MainThreadTicker]` for the global main thread executor. - /// Use this to tick the main thread executor. - /// Returns None if called on not the main thread. + /// Gets the `[MainThreadTicker]` for this executor. + /// Use this to tick the executor. + /// It only returns the ticker if it's on the thread the executor was created on + /// and returns `None` otherwise. pub fn ticker(&self) -> Option { - // always return ticker when testing to allow tests to run off main thread - dbg!("hjj"); - #[cfg(test)] - if true { - dbg!("blah"); + if thread::current().id() == self.thread_id { return Some(MainThreadTicker { - executor: self.0.clone(), + executor: self.executor.clone(), _marker: PhantomData::default(), }); } - - if let Some(is_main) = is_main_thread() { - if is_main { - return Some(MainThreadTicker { - executor: self.0.clone(), - _marker: PhantomData::default(), - }); - } - } None } } diff --git a/crates/bevy_tasks/src/slice.rs b/crates/bevy_tasks/src/slice.rs index 4b5d875ea989b..44372304e71d5 100644 --- a/crates/bevy_tasks/src/slice.rs +++ b/crates/bevy_tasks/src/slice.rs @@ -37,7 +37,7 @@ pub trait ParallelSlice: AsRef<[T]> { { let slice = self.as_ref(); let f = &f; - task_pool.scope(|scope| { + task_pool.scope(None, |scope| { for chunk in slice.chunks(chunk_size) { scope.spawn(async move { f(chunk) }); } @@ -134,7 +134,7 @@ pub trait ParallelSliceMut: AsMut<[T]> { { let slice = self.as_mut(); let f = &f; - task_pool.scope(|scope| { + task_pool.scope(None, |scope| { for chunk in slice.chunks_mut(chunk_size) { scope.spawn(async move { f(chunk) }); } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 02126b8f3dd87..1297b422e63ac 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -11,7 +11,7 @@ use concurrent_queue::ConcurrentQueue; use futures_lite::{future, FutureExt}; use crate::Task; -use crate::{main_thread_executor::MainThreadSpawner, MainThreadExecutor}; +use crate::{main_thread_executor::MainThreadSpawner, ThreadExecutor}; /// Used to create a [`TaskPool`] #[derive(Debug, Default, Clone)] @@ -165,7 +165,7 @@ impl TaskPool { /// /// let pool = TaskPool::new(); /// let mut x = 0; - /// let results = pool.scope(|s| { + /// let results = pool.scope(None, |s| { /// s.spawn(async { /// // you can borrow the spawner inside a task and spawn tasks from within the task /// s.spawn(async { @@ -185,7 +185,7 @@ impl TaskPool { /// assert!(results.contains(&1)); /// /// // The ordering is deterministic if you only spawn directly from the closure function. - /// let results = pool.scope(|s| { + /// let results = pool.scope(None, |s| { /// s.spawn(async { 0 }); /// s.spawn(async { 1 }); /// }); @@ -210,7 +210,7 @@ impl TaskPool { /// fn scope_escapes_closure() { /// let pool = TaskPool::new(); /// let foo = Box::new(42); - /// pool.scope(|scope| { + /// pool.scope(None, |scope| { /// std::thread::spawn(move || { /// // UB. This could spawn on the scope after `.scope` returns and the internal Scope is dropped. /// scope.spawn(async move { @@ -225,7 +225,7 @@ impl TaskPool { /// use bevy_tasks::TaskPool; /// fn cannot_borrow_from_closure() { /// let pool = TaskPool::new(); - /// pool.scope(|scope| { + /// pool.scope(None, |scope| { /// let x = 1; /// let y = &x; /// scope.spawn(async move { @@ -234,7 +234,7 @@ impl TaskPool { /// }); /// } /// - pub fn scope<'env, F, T>(&self, f: F) -> Vec + pub fn scope<'env, F, T>(&self, thread_executor: Option>, f: F) -> Vec where F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, @@ -247,23 +247,20 @@ impl TaskPool { let executor: &async_executor::Executor = &self.executor; let executor: &'env async_executor::Executor = unsafe { mem::transmute(executor) }; - #[cfg(not(test))] - let main_thread_executor = MainThreadExecutor::init(); - // for testing configure a new instance of main thread executor for every scope - // this helps us pretend that the thread that an app or stage is constructed on is the main thread - #[cfg(test)] - let main_thread_executor = MainThreadExecutor(Arc::new(async_executor::Executor::new())); - - let main_thread_spawner = main_thread_executor.spawner(); - let main_thread_spawner: MainThreadSpawner<'env> = - unsafe { mem::transmute(main_thread_spawner) }; - let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); + let thread_executor = if let Some(thread_executor) = thread_executor { + thread_executor + } else { + Arc::new(ThreadExecutor::new()) + }; + let thread_spawner = thread_executor.spawner(); + let thread_spawner: MainThreadSpawner<'env> = unsafe { mem::transmute(thread_spawner) }; + let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); let spawned_ref: &'env ConcurrentQueue> = unsafe { mem::transmute(&spawned) }; let scope = Scope { executor, - main_thread_spawner, + thread_spawner, spawned: spawned_ref, scope: PhantomData, env: PhantomData, @@ -286,10 +283,9 @@ impl TaskPool { results }; - if let Some(main_thread_ticker) = main_thread_executor.ticker() { + if let Some(main_thread_ticker) = thread_executor.ticker() { let tick_forever = async move { loop { - dbg!("tivk"); main_thread_ticker.tick().await; } }; @@ -371,7 +367,7 @@ impl Drop for TaskPool { #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope async_executor::Executor<'scope>, - main_thread_spawner: MainThreadSpawner<'scope>, + thread_spawner: MainThreadSpawner<'scope>, spawned: &'scope ConcurrentQueue>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, @@ -400,10 +396,8 @@ impl<'scope, 'env, T: Send + 'scope> Scope<'scope, 'env, T> { /// [`Scope::spawn`] instead, unless the provided future needs to run on the scope's thread. /// /// For more information, see [`TaskPool::scope`]. - pub fn spawn_on_main + 'scope + Send>(&self, f: Fut) { - let main_thread_spawner: &MainThreadSpawner<'scope> = - unsafe { mem::transmute(&self.main_thread_spawner) }; - let task = main_thread_spawner.spawn(f).fallible(); + pub fn spawn_on_scope + 'scope + Send>(&self, f: Fut) { + let task = self.thread_spawner.spawn(f).fallible(); // ConcurrentQueue only errors when closed or full, but we never // close and use an unbouded queue, so it is safe to unwrap self.spawned.push(task).unwrap(); @@ -441,7 +435,7 @@ mod tests { let count = Arc::new(AtomicI32::new(0)); - let outputs = pool.scope(|scope| { + let outputs = pool.scope(None, |scope| { for _ in 0..100 { let count_clone = count.clone(); scope.spawn(async move { @@ -473,7 +467,7 @@ mod tests { let local_count = Arc::new(AtomicI32::new(0)); let non_local_count = Arc::new(AtomicI32::new(0)); - let outputs = pool.scope(|scope| { + let outputs = pool.scope(None, |scope| { for i in 0..100 { if i % 2 == 0 { let count_clone = non_local_count.clone(); @@ -487,7 +481,7 @@ mod tests { }); } else { let count_clone = local_count.clone(); - scope.spawn_on_main(async move { + scope.spawn_on_scope(async move { if *foo != 42 { panic!("not 42!?!?") } else { @@ -521,14 +515,14 @@ mod tests { let inner_pool = pool.clone(); let inner_thread_check_failed = thread_check_failed.clone(); std::thread::spawn(move || { - inner_pool.scope(|scope| { + inner_pool.scope(None, |scope| { let inner_count_clone = count_clone.clone(); scope.spawn(async move { inner_count_clone.fetch_add(1, Ordering::Release); }); let spawner = std::thread::current().id(); let inner_count_clone = count_clone.clone(); - scope.spawn_on_main(async move { + scope.spawn_on_scope(async move { inner_count_clone.fetch_add(1, Ordering::Release); if std::thread::current().id() != spawner { // NOTE: This check is using an atomic rather than simply panicing the @@ -554,7 +548,7 @@ mod tests { let count = Arc::new(AtomicI32::new(0)); - let outputs: Vec = pool.scope(|scope| { + let outputs: Vec = pool.scope(None, |scope| { for _ in 0..10 { let count_clone = count.clone(); scope.spawn(async move { @@ -596,14 +590,14 @@ mod tests { let inner_pool = pool.clone(); let inner_thread_check_failed = thread_check_failed.clone(); std::thread::spawn(move || { - inner_pool.scope(|scope| { + inner_pool.scope(None, |scope| { let spawner = std::thread::current().id(); let inner_count_clone = count_clone.clone(); scope.spawn(async move { inner_count_clone.fetch_add(1, Ordering::Release); // spawning on the scope from another thread runs the futures on the scope's thread - scope.spawn_on_main(async move { + scope.spawn_on_scope(async move { inner_count_clone.fetch_add(1, Ordering::Release); if std::thread::current().id() != spawner { // NOTE: This check is using an atomic rather than simply panicing the From d971288de93e2c9f673cd21816724fb2593ee80f Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 7 Nov 2022 20:07:08 -0800 Subject: [PATCH 10/55] add a test and update docs --- crates/bevy_tasks/Cargo.toml | 1 - crates/bevy_tasks/src/main_thread_executor.rs | 33 +++++++++++++++---- crates/bevy_tasks/src/task_pool.rs | 5 +++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/crates/bevy_tasks/Cargo.toml b/crates/bevy_tasks/Cargo.toml index 291456b5d02b5..e39e182dffedc 100644 --- a/crates/bevy_tasks/Cargo.toml +++ b/crates/bevy_tasks/Cargo.toml @@ -15,7 +15,6 @@ async-channel = "1.4.2" async-task = "4.2.0" once_cell = "1.7" concurrent-queue = "2.0.0" -is_main_thread = "0.1.0" [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = "0.4" diff --git a/crates/bevy_tasks/src/main_thread_executor.rs b/crates/bevy_tasks/src/main_thread_executor.rs index 2e39cc230b778..eceb1b5d5c884 100644 --- a/crates/bevy_tasks/src/main_thread_executor.rs +++ b/crates/bevy_tasks/src/main_thread_executor.rs @@ -7,11 +7,9 @@ use std::{ use async_executor::{Executor, Task}; use futures_lite::Future; -/// Use to access the global main thread executor. Be aware that the main thread executor -/// only makes progress when it is ticked. This normally happens in `[TaskPool::scope]`. +/// An executor that can only be ticked on the thread it was instantiated on. #[derive(Debug)] pub struct ThreadExecutor { - // this is only pub crate for testing purposes, do not contruct otherwise executor: Arc>, thread_id: ThreadId, } @@ -26,13 +24,13 @@ impl Default for ThreadExecutor { } impl ThreadExecutor { - /// Initializes the global `[MainThreadExecutor]` instance. + /// createa a new `[ThreadExecutor]` pub fn new() -> Self { Self::default() } - /// Gets the `[MainThreadSpawner]` for the global main thread executor. - /// Use this to spawn tasks on the main thread. + /// Gets the `[MainThreadSpawner]` for the thread executor. + /// Use this to spawn tasks that run on the thread this was instatiated on. pub fn spawner(&self) -> MainThreadSpawner<'static> { MainThreadSpawner(self.executor.clone()) } @@ -69,8 +67,29 @@ pub struct MainThreadTicker { } impl MainThreadTicker { /// Tick the main thread executor. - /// This needs to be called manually on the main thread if a `[TaskPool::scope]` is not active + /// This needs to be called manually on the thread if it is not being used with + /// a `[TaskPool::scope]`. pub fn tick(&self) -> impl Future + '_ { self.executor.tick() } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + #[test] + fn test_ticker() { + let executor = Arc::new(ThreadExecutor::new()); + let ticker = executor.ticker(); + assert!(ticker.is_some()); + + std::thread::scope(|s| { + s.spawn(|| { + let ticker = executor.ticker(); + assert!(ticker.is_none()); + }); + }); + } +} diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 1297b422e63ac..2e74093fd0f6d 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -158,6 +158,11 @@ impl TaskPool { /// /// This is similar to `rayon::scope` and `crossbeam::scope` /// + /// The `thread_executor` optional parameter can be used to pass a `[ThreadExecutor]` to + /// spawn tasks on when calling `spawn_on_scope`. This can be useful for spawning tasks that + /// must run on the main thread. If `None` is passed then `spawn_on_scope` runs tasks on + /// the thread `scope` is run on. + /// /// # Example /// /// ``` From bbd8626b6e1255e0b730c198b368db093feeaec2 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 7 Nov 2022 20:41:27 -0800 Subject: [PATCH 11/55] fix wasm compilation probably doesn't run correctly --- crates/bevy_tasks/src/lib.rs | 4 +++- crates/bevy_tasks/src/single_threaded_task_pool.rs | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index ce35e8ee4b54f..e3e0250a2e568 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -15,14 +15,16 @@ pub use task_pool::{Scope, TaskPool, TaskPoolBuilder}; #[cfg(target_arch = "wasm32")] mod single_threaded_task_pool; #[cfg(target_arch = "wasm32")] -pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder}; +pub use single_threaded_task_pool::{Scope, TaskPool, TaskPoolBuilder, ThreadExecutor}; mod usages; #[cfg(not(target_arch = "wasm32"))] pub use usages::tick_global_task_pools_on_main_thread; pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; +#[cfg(not(target_arch = "wasm32"))] mod main_thread_executor; +#[cfg(not(target_arch = "wasm32"))] pub use main_thread_executor::ThreadExecutor; mod iter; diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 8fa37f4f2361b..cdcfdaa7c05c7 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -9,6 +9,16 @@ use std::{ #[derive(Debug, Default, Clone)] pub struct TaskPoolBuilder {} +/// dummy struct for wasm +#[derive(Default)] +pub struct ThreadExecutor; +impl ThreadExecutor { + /// creates a new `ThreadExecutor` + pub fn new() -> Self { + Self + } +} + impl TaskPoolBuilder { /// Creates a new TaskPoolBuilder instance pub fn new() -> Self { @@ -62,7 +72,7 @@ impl TaskPool { /// to spawn tasks. This function will await the completion of all tasks before returning. /// /// This is similar to `rayon::scope` and `crossbeam::scope` - pub fn scope<'env, F, T>(&self, f: F) -> Vec + pub fn scope<'env, F, T>(&self, _thread_executor: Option>, f: F) -> Vec where F: for<'scope> FnOnce(&'env mut Scope<'scope, 'env, T>), T: Send + 'static, From 6842f4e9f0eed2d432256bedd36ab9a893edf43a Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Tue, 8 Nov 2022 17:40:15 -0800 Subject: [PATCH 12/55] move extract after scope to make input processing closer to main app run --- crates/bevy_app/src/app.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 59fe7c8271ba3..fe605159b5d8a 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -157,15 +157,15 @@ impl App { .map(|e| e.0.clone()); ComputeTaskPool::init(TaskPool::default).scope(thread_executor, |scope| { if self.run_once { - for sub_app in self.sub_apps.values_mut() { - (sub_app.extract)(&mut self.world, &mut sub_app.app); - } for sub_app in self.sub_apps.values_mut() { scope.spawn(async { (sub_app.runner)(&mut sub_app.app) }); } } self.schedule.run(&mut self.world); }); + for sub_app in self.sub_apps.values_mut() { + (sub_app.extract)(&mut self.world, &mut sub_app.app); + } self.run_once = true; self.world.clear_trackers(); From b6598fb9567564e1995780645189839bf407eec3 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Wed, 9 Nov 2022 10:28:55 -0800 Subject: [PATCH 13/55] switch to sending render world back and forth with channels --- crates/bevy_app/src/app.rs | 41 +++++---- crates/bevy_render/Cargo.toml | 2 + crates/bevy_render/src/lib.rs | 1 + crates/bevy_render/src/pipelined_rendering.rs | 85 +++++++++++++++++++ crates/bevy_winit/Cargo.toml | 1 + crates/bevy_winit/src/lib.rs | 6 +- 6 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 crates/bevy_render/src/pipelined_rendering.rs diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index fe605159b5d8a..a87cb5021dfc7 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -10,7 +10,6 @@ use bevy_ecs::{ system::Resource, world::World, }; -use bevy_tasks::{ComputeTaskPool, TaskPool}; use bevy_utils::{tracing::debug, HashMap, HashSet}; use std::fmt::Debug; @@ -74,7 +73,6 @@ pub struct App { sub_apps: HashMap, plugin_registry: Vec>, plugin_name_added: HashSet, - run_once: bool, } impl Debug for App { @@ -88,12 +86,24 @@ impl Debug for App { } /// Each `SubApp` has its own [`Schedule`] and [`World`], enabling a separation of concerns. -struct SubApp { +pub struct SubApp { app: App, extract: Box, // Send + Sync bound is only required to make SubApp send sync runner: Box, // this send sync bound is required since we're actually sending this function to another thread } +impl SubApp { + /// runs the `SubApp` with its runner + pub fn run(&mut self) { + (self.runner)(&mut self.app); + } + + /// extract data from main world to sub app + pub fn extract(&mut self, main_world: &mut World) { + (self.extract)(main_world, &mut self.app); + } +} + impl Debug for SubApp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "SubApp {{ app: ")?; @@ -139,7 +149,6 @@ impl App { sub_apps: HashMap::default(), plugin_registry: Vec::default(), plugin_name_added: Default::default(), - run_once: false, } } @@ -151,22 +160,10 @@ impl App { pub fn update(&mut self) { #[cfg(feature = "trace")] let _bevy_frame_update_span = info_span!("frame").entered(); - let thread_executor = self - .world - .get_resource::() - .map(|e| e.0.clone()); - ComputeTaskPool::init(TaskPool::default).scope(thread_executor, |scope| { - if self.run_once { - for sub_app in self.sub_apps.values_mut() { - scope.spawn(async { (sub_app.runner)(&mut sub_app.app) }); - } - } - self.schedule.run(&mut self.world); - }); + self.schedule.run(&mut self.world); for sub_app in self.sub_apps.values_mut() { (sub_app.extract)(&mut self.world, &mut sub_app.app); } - self.run_once = true; self.world.clear_trackers(); } @@ -1057,6 +1054,16 @@ impl App { } } + /// inserts an existing sub app into the app + pub fn insert_sub_app(&mut self, label: impl AppLabel, sub_app: SubApp) { + self.sub_apps.insert(label.as_label(), sub_app); + } + + /// remove a sub app from the app + pub fn remove_sub_app(&mut self, label: impl AppLabel) -> Option { + self.sub_apps.remove(&label.as_label()) + } + /// Retrieves a `SubApp` inside this [`App`] with the given label, if it exists. Otherwise returns /// an [`Err`] containing the given label. pub fn get_sub_app(&self, label: impl AppLabel) -> Result<&App, impl AppLabel> { diff --git a/crates/bevy_render/Cargo.toml b/crates/bevy_render/Cargo.toml index 605871172c2ec..1df10c205599c 100644 --- a/crates/bevy_render/Cargo.toml +++ b/crates/bevy_render/Cargo.toml @@ -44,6 +44,7 @@ bevy_time = { path = "../bevy_time", version = "0.9.0" } bevy_transform = { path = "../bevy_transform", version = "0.9.0" } bevy_window = { path = "../bevy_window", version = "0.9.0" } bevy_utils = { path = "../bevy_utils", version = "0.9.0" } +bevy_tasks = { path = "../bevy_tasks", version = "0.9.0" } # rendering image = { version = "0.24", default-features = false } @@ -75,3 +76,4 @@ basis-universal = { version = "0.2.0", optional = true } encase = { version = "0.4", features = ["glam"] } # For wgpu profiling using tracing. Use `RUST_LOG=info` to also capture the wgpu spans. profiling = { version = "1", features = ["profile-with-tracing"], optional = true } +async-channel = "1.4" \ No newline at end of file diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index cd2fbe0cad957..400e2dc576f8d 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -7,6 +7,7 @@ mod extract_param; pub mod extract_resource; pub mod globals; pub mod mesh; +pub mod pipelined_rendering; pub mod primitives; pub mod rangefinder; pub mod render_asset; diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs new file mode 100644 index 0000000000000..31bb22aea2ee4 --- /dev/null +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -0,0 +1,85 @@ +use async_channel::{Receiver, Sender}; +use bevy_app::{App, SubApp}; +use bevy_ecs::{schedule::MainThreadExecutor, system::Resource, world::Mut}; +use bevy_tasks::ComputeTaskPool; + +#[cfg(feature = "trace")] +use bevy_utils::tracing::Instrument; + +use crate::RenderApp; + +/// Resource to be used for pipelined rendering for sending the render app from the main thread to the rendering thread +#[derive(Resource)] +pub struct MainToRenderAppSender(pub Sender); + +/// Resource used by pipelined rendering to send the render app from the render thread to the main thread +#[derive(Resource)] +pub struct RenderToMainAppReceiver(pub Receiver); + +/// sets up the render thread and insert resource into the main app for controlling the render thread +pub fn setup_pipelined_rendering(app: &mut App) { + let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::(1); + let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::(1); + + let render_app = app.remove_sub_app(RenderApp).unwrap(); + render_to_app_sender.send_blocking(render_app).unwrap(); + + app.insert_resource(MainToRenderAppSender(app_to_render_sender)); + app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver)); + + let render_task = async move { + loop { + // TODO: exit loop when app is exited + let recv_task = app_to_render_receiver.recv(); + #[cfg(feature = "trace")] + let span = bevy_utils::tracing::info_span!("receive render world from main"); + #[cfg(feature = "trace")] + let recv_task = recv_task.instrument(span); + let mut sub_app = recv_task.await.unwrap(); + sub_app.run(); + render_to_app_sender.send(sub_app).await.unwrap(); + } + }; + #[cfg(feature = "trace")] + let span = bevy_utils::tracing::info_span!("render task"); + #[cfg(feature = "trace")] + let render_task = render_task.instrument(span); + ComputeTaskPool::get().spawn(render_task).detach(); +} + +pub fn update_rendering(app: &mut App) { + app.update(); + + // wait to get the render app back to signal that rendering is finished + let mut render_app = app + .world + .resource_scope(|world, main_thread_executor: Mut| { + ComputeTaskPool::get() + .scope(Some(main_thread_executor.0.clone()), |s| { + s.spawn(async { + let receiver = world.get_resource::().unwrap(); + let recv = receiver.0.recv(); + #[cfg(feature = "trace")] + let span = bevy_utils::tracing::info_span!("wait for render"); + #[cfg(feature = "trace")] + let recv = recv.instrument(span); + recv.await.unwrap() + }); + }) + .pop() + }) + .unwrap(); + + render_app.extract(&mut app.world); + + { + #[cfg(feature = "trace")] + let _span = bevy_utils::tracing::info_span!("send world to render").entered(); + app.world + .resource_scope(|_world, sender: Mut| { + sender.0.send_blocking(render_app).unwrap(); + }); + } + + // frame pacing plugin should run here somehow. i.e. after rendering, but before input handling +} diff --git a/crates/bevy_winit/Cargo.toml b/crates/bevy_winit/Cargo.toml index 96864623f1113..d9100767c88b1 100644 --- a/crates/bevy_winit/Cargo.toml +++ b/crates/bevy_winit/Cargo.toml @@ -15,6 +15,7 @@ x11 = ["winit/x11"] [dependencies] # bevy +bevy_render = { path = "../bevy_render", version = "0.9.0-dev" } bevy_app = { path = "../bevy_app", version = "0.9.0" } bevy_ecs = { path = "../bevy_ecs", version = "0.9.0" } bevy_input = { path = "../bevy_input", version = "0.9.0" } diff --git a/crates/bevy_winit/src/lib.rs b/crates/bevy_winit/src/lib.rs index 749d256882500..d62845fd78732 100644 --- a/crates/bevy_winit/src/lib.rs +++ b/crates/bevy_winit/src/lib.rs @@ -4,6 +4,7 @@ mod web_resize; mod winit_config; mod winit_windows; +use bevy_render::pipelined_rendering::{setup_pipelined_rendering, update_rendering}; use winit::window::CursorGrabMode; pub use winit_config::*; pub use winit_windows::*; @@ -368,6 +369,8 @@ pub fn winit_runner_with(mut app: App) { let return_from_run = app.world.resource::().return_from_run; + setup_pipelined_rendering(&mut app); + trace!("Entering winit event loop"); let event_handler = move |event: Event<()>, @@ -607,7 +610,8 @@ pub fn winit_runner_with(mut app: App) { }; if update { winit_state.last_update = Instant::now(); - app.update(); + update_rendering(&mut app); + // app.update(); } } Event::RedrawEventsCleared => { From 0892f5ba2fed7a26117ee03164fb0e5679136a3c Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Wed, 9 Nov 2022 13:47:41 -0800 Subject: [PATCH 14/55] add ability to disable pipelined rendering --- crates/bevy_app/src/app.rs | 5 +- crates/bevy_render/src/lib.rs | 35 ++++++++++++-- crates/bevy_render/src/pipelined_rendering.rs | 46 ++++++++----------- crates/bevy_winit/src/lib.rs | 7 ++- 4 files changed, 57 insertions(+), 36 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index a87cb5021dfc7..b0b3df5b0f5b3 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -159,10 +159,11 @@ impl App { /// See [`add_sub_app`](Self::add_sub_app) and [`run_once`](Schedule::run_once) for more details. pub fn update(&mut self) { #[cfg(feature = "trace")] - let _bevy_frame_update_span = info_span!("frame").entered(); + let _bevy_frame_update_span = info_span!("main_app").entered(); self.schedule.run(&mut self.world); for sub_app in self.sub_apps.values_mut() { - (sub_app.extract)(&mut self.world, &mut sub_app.app); + sub_app.extract(&mut self.world); + sub_app.run(); } self.world.clear_trackers(); diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 400e2dc576f8d..dc9d6de2d43fa 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -38,6 +38,8 @@ pub mod prelude { use globals::GlobalsPlugin; pub use once_cell; +use pipelined_rendering::update_rendering; +use prelude::ComputedVisibility; use crate::{ camera::CameraPlugin, @@ -56,8 +58,20 @@ use std::{ }; /// Contains the default Bevy rendering backend based on wgpu. -#[derive(Default)] -pub struct RenderPlugin; +pub struct RenderPlugin { + pub use_pipelined_rendering: bool, +} + +impl Default for RenderPlugin { + fn default() -> Self { + RenderPlugin { + #[cfg(not(target_arch = "wasm32"))] + use_pipelined_rendering: true, + #[cfg(target_arch = "wasm32")] + use_pipelined_rendering: false, + } + } +} /// The labels of the default App rendering stages. #[derive(Debug, Hash, PartialEq, Eq, Clone, StageLabel)] @@ -122,6 +136,10 @@ pub mod main_graph { #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] pub struct RenderApp; +/// A Label for the sub app that runs the parts of pipelined rendering that need to run on the main thread. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] +pub struct PipelinedRenderingApp; + impl Plugin for RenderPlugin { /// Initializes the renderer, sets up the [`RenderStage`](RenderStage) and creates the rendering sub-app. fn build(&self, app: &mut App) { @@ -212,7 +230,7 @@ impl Plugin for RenderPlugin { app.add_sub_app(RenderApp, render_app, move |app_world, render_app| { #[cfg(feature = "trace")] - let _render_span = bevy_utils::tracing::info_span!("renderer subapp").entered(); + let _render_span = bevy_utils::tracing::info_span!("extract").entered(); { #[cfg(feature = "trace")] let _stage_span = @@ -319,6 +337,17 @@ impl Plugin for RenderPlugin { render_app.world.clear_entities(); } }); + + if self.use_pipelined_rendering { + app.add_sub_app( + PipelinedRenderingApp, + App::new(), + |app_world, _render_app| { + update_rendering(app_world); + }, + |_render_world| {}, + ); + } } app.add_plugin(ValidParentCheckPlugin::::default()) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 31bb22aea2ee4..3227cb014adb5 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -1,12 +1,16 @@ use async_channel::{Receiver, Sender}; use bevy_app::{App, SubApp}; -use bevy_ecs::{schedule::MainThreadExecutor, system::Resource, world::Mut}; +use bevy_ecs::{ + schedule::MainThreadExecutor, + system::Resource, + world::{Mut, World}, +}; use bevy_tasks::ComputeTaskPool; #[cfg(feature = "trace")] use bevy_utils::tracing::Instrument; -use crate::RenderApp; +use crate::{PipelinedRenderingApp, RenderApp}; /// Resource to be used for pipelined rendering for sending the render app from the main thread to the rendering thread #[derive(Resource)] @@ -17,7 +21,12 @@ pub struct MainToRenderAppSender(pub Sender); pub struct RenderToMainAppReceiver(pub Receiver); /// sets up the render thread and insert resource into the main app for controlling the render thread -pub fn setup_pipelined_rendering(app: &mut App) { +pub fn setup_rendering(app: &mut App) { + // skip this if pipelined rendering is not enabled + if app.get_sub_app(PipelinedRenderingApp).is_err() { + return; + } + let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::(1); let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::(1); @@ -31,38 +40,27 @@ pub fn setup_pipelined_rendering(app: &mut App) { loop { // TODO: exit loop when app is exited let recv_task = app_to_render_receiver.recv(); - #[cfg(feature = "trace")] - let span = bevy_utils::tracing::info_span!("receive render world from main"); - #[cfg(feature = "trace")] - let recv_task = recv_task.instrument(span); let mut sub_app = recv_task.await.unwrap(); sub_app.run(); render_to_app_sender.send(sub_app).await.unwrap(); } }; #[cfg(feature = "trace")] - let span = bevy_utils::tracing::info_span!("render task"); + let span = bevy_utils::tracing::info_span!("render app"); #[cfg(feature = "trace")] let render_task = render_task.instrument(span); ComputeTaskPool::get().spawn(render_task).detach(); } -pub fn update_rendering(app: &mut App) { - app.update(); - +pub fn update_rendering(app_world: &mut World) { // wait to get the render app back to signal that rendering is finished - let mut render_app = app - .world + let mut render_app = app_world .resource_scope(|world, main_thread_executor: Mut| { ComputeTaskPool::get() .scope(Some(main_thread_executor.0.clone()), |s| { s.spawn(async { let receiver = world.get_resource::().unwrap(); let recv = receiver.0.recv(); - #[cfg(feature = "trace")] - let span = bevy_utils::tracing::info_span!("wait for render"); - #[cfg(feature = "trace")] - let recv = recv.instrument(span); recv.await.unwrap() }); }) @@ -70,16 +68,10 @@ pub fn update_rendering(app: &mut App) { }) .unwrap(); - render_app.extract(&mut app.world); - - { - #[cfg(feature = "trace")] - let _span = bevy_utils::tracing::info_span!("send world to render").entered(); - app.world - .resource_scope(|_world, sender: Mut| { - sender.0.send_blocking(render_app).unwrap(); - }); - } + render_app.extract(app_world); + app_world.resource_scope(|_world, sender: Mut| { + sender.0.send_blocking(render_app).unwrap(); + }); // frame pacing plugin should run here somehow. i.e. after rendering, but before input handling } diff --git a/crates/bevy_winit/src/lib.rs b/crates/bevy_winit/src/lib.rs index d62845fd78732..32b301f7f2124 100644 --- a/crates/bevy_winit/src/lib.rs +++ b/crates/bevy_winit/src/lib.rs @@ -4,7 +4,7 @@ mod web_resize; mod winit_config; mod winit_windows; -use bevy_render::pipelined_rendering::{setup_pipelined_rendering, update_rendering}; +use bevy_render::pipelined_rendering::setup_rendering; use winit::window::CursorGrabMode; pub use winit_config::*; pub use winit_windows::*; @@ -369,7 +369,7 @@ pub fn winit_runner_with(mut app: App) { let return_from_run = app.world.resource::().return_from_run; - setup_pipelined_rendering(&mut app); + setup_rendering(&mut app); trace!("Entering winit event loop"); @@ -610,8 +610,7 @@ pub fn winit_runner_with(mut app: App) { }; if update { winit_state.last_update = Instant::now(); - update_rendering(&mut app); - // app.update(); + app.update(); } } Event::RedrawEventsCleared => { From dd9163ccd98e14ab7094634bdbe3cefa90916c00 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Wed, 9 Nov 2022 18:11:06 -0800 Subject: [PATCH 15/55] cleanup --- crates/bevy_app/src/app.rs | 20 +++--- .../src/schedule/executor_parallel.rs | 2 +- crates/bevy_render/Cargo.toml | 2 +- crates/bevy_render/src/lib.rs | 7 +- crates/bevy_render/src/pipelined_rendering.rs | 70 +++++++++---------- crates/bevy_tasks/src/main_thread_executor.rs | 18 ++--- crates/bevy_tasks/src/task_pool.rs | 8 +-- 7 files changed, 64 insertions(+), 63 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index b0b3df5b0f5b3..7668786a549e7 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -67,7 +67,7 @@ pub struct App { /// the application's event loop and advancing the [`Schedule`]. /// Typically, it is not configured manually, but set by one of Bevy's built-in plugins. /// See `bevy::winit::WinitPlugin` and [`ScheduleRunnerPlugin`](crate::schedule_runner::ScheduleRunnerPlugin). - pub runner: Box, // send/sync bound is only required to make App Send/Sync + pub runner: Box, // Send + Sync bound is only required to make App Send + Sync /// A container of [`Stage`]s set to be run in a linear order. pub schedule: Schedule, sub_apps: HashMap, @@ -89,7 +89,7 @@ impl Debug for App { pub struct SubApp { app: App, extract: Box, // Send + Sync bound is only required to make SubApp send sync - runner: Box, // this send sync bound is required since we're actually sending this function to another thread + runner: Box, // this Send + Sync bound is required since we're running this function on another thread } impl SubApp { @@ -159,7 +159,7 @@ impl App { /// See [`add_sub_app`](Self::add_sub_app) and [`run_once`](Schedule::run_once) for more details. pub fn update(&mut self) { #[cfg(feature = "trace")] - let _bevy_frame_update_span = info_span!("main_app").entered(); + let _bevy_frame_update_span = info_span!("main app").entered(); self.schedule.run(&mut self.world); for sub_app in self.sub_apps.values_mut() { sub_app.extract(&mut self.world); @@ -997,15 +997,15 @@ impl App { /// Adds an [`App`] as a child of the current one. /// - /// The provided function `f` is called by the [`update`](Self::update) method. The [`World`] + /// The provided functions `extract` and `runner` are normally called by the [`update`](Self::update) method. The [`World`] /// parameter represents the main app world, while the [`App`] parameter is just a mutable /// reference to the `SubApp` itself. pub fn add_sub_app( &mut self, label: impl AppLabel, mut app: App, - sub_app_extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, - sub_app_runner: impl Fn(&mut App) + 'static + Send + Sync, + extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, + runner: impl Fn(&mut App) + 'static + Send + Sync, ) -> &mut Self { if let Some(executor) = self.world.get_resource::() { app.world.insert_resource(executor.clone()); @@ -1014,8 +1014,8 @@ impl App { label.as_label(), SubApp { app, - extract: Box::new(sub_app_extract), - runner: Box::new(sub_app_runner), + extract: Box::new(extract), + runner: Box::new(runner), }, ); self @@ -1055,12 +1055,12 @@ impl App { } } - /// inserts an existing sub app into the app + /// Inserts an existing sub app into the app pub fn insert_sub_app(&mut self, label: impl AppLabel, sub_app: SubApp) { self.sub_apps.insert(label.as_label(), sub_app); } - /// remove a sub app from the app + /// Removes a sub app from the app. Returns None if the label doesn't exist. pub fn remove_sub_app(&mut self, label: impl AppLabel) -> Option { self.sub_apps.remove(&label.as_label()) } diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index 5a80aea31307a..0ff8dd1fd90e9 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -18,7 +18,7 @@ use fixedbitset::FixedBitSet; #[cfg(test)] use scheduling_event::*; -/// +/// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Default)] pub struct MainThreadExecutor(pub Arc); diff --git a/crates/bevy_render/Cargo.toml b/crates/bevy_render/Cargo.toml index 1df10c205599c..d267f4a0ae284 100644 --- a/crates/bevy_render/Cargo.toml +++ b/crates/bevy_render/Cargo.toml @@ -76,4 +76,4 @@ basis-universal = { version = "0.2.0", optional = true } encase = { version = "0.4", features = ["glam"] } # For wgpu profiling using tracing. Use `RUST_LOG=info` to also capture the wgpu spans. profiling = { version = "1", features = ["profile-with-tracing"], optional = true } -async-channel = "1.4" \ No newline at end of file +async-channel = "1.4" diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index dc9d6de2d43fa..e8c5d1142ab10 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -59,6 +59,9 @@ use std::{ /// Contains the default Bevy rendering backend based on wgpu. pub struct RenderPlugin { + /// Pipelined rendering runs the rendering simultaneously with the main app. + /// Use this to turn pipelined rendering on or off. By default it's on in native + /// environments and off in wasm. pub use_pipelined_rendering: bool, } @@ -230,7 +233,7 @@ impl Plugin for RenderPlugin { app.add_sub_app(RenderApp, render_app, move |app_world, render_app| { #[cfg(feature = "trace")] - let _render_span = bevy_utils::tracing::info_span!("extract").entered(); + let _render_span = bevy_utils::tracing::info_span!("extract main to render app").entered(); { #[cfg(feature = "trace")] let _stage_span = @@ -265,6 +268,8 @@ impl Plugin for RenderPlugin { extract(app_world, render_app); } }, |render_app| { + #[cfg(feature = "trace")] + let _render_span = bevy_utils::tracing::info_span!("render app").entered(); { #[cfg(feature = "trace")] let _stage_span = diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 3227cb014adb5..0f71b58cb955e 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -1,4 +1,5 @@ use async_channel::{Receiver, Sender}; + use bevy_app::{App, SubApp}; use bevy_ecs::{ schedule::MainThreadExecutor, @@ -7,22 +8,19 @@ use bevy_ecs::{ }; use bevy_tasks::ComputeTaskPool; -#[cfg(feature = "trace")] -use bevy_utils::tracing::Instrument; - use crate::{PipelinedRenderingApp, RenderApp}; -/// Resource to be used for pipelined rendering for sending the render app from the main thread to the rendering thread +/// Resource for pipelined rendering to send the render app from the main thread to the rendering thread #[derive(Resource)] pub struct MainToRenderAppSender(pub Sender); -/// Resource used by pipelined rendering to send the render app from the render thread to the main thread +/// Resource for pipelined rendering to send the render app from the render thread to the main thread #[derive(Resource)] pub struct RenderToMainAppReceiver(pub Receiver); -/// sets up the render thread and insert resource into the main app for controlling the render thread +/// Sets up the render thread and inserts resources into the main app used for controlling the render thread +/// This does nothing if pipelined rendering is not enabled. pub fn setup_rendering(app: &mut App) { - // skip this if pipelined rendering is not enabled if app.get_sub_app(PipelinedRenderingApp).is_err() { return; } @@ -36,42 +34,38 @@ pub fn setup_rendering(app: &mut App) { app.insert_resource(MainToRenderAppSender(app_to_render_sender)); app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver)); - let render_task = async move { - loop { - // TODO: exit loop when app is exited - let recv_task = app_to_render_receiver.recv(); - let mut sub_app = recv_task.await.unwrap(); - sub_app.run(); - render_to_app_sender.send(sub_app).await.unwrap(); - } - }; - #[cfg(feature = "trace")] - let span = bevy_utils::tracing::info_span!("render app"); - #[cfg(feature = "trace")] - let render_task = render_task.instrument(span); - ComputeTaskPool::get().spawn(render_task).detach(); + ComputeTaskPool::get() + .spawn(async move { + loop { + // TODO: exit loop when app is exited + let recv_task = app_to_render_receiver.recv(); + let mut sub_app = recv_task.await.unwrap(); + sub_app.run(); + render_to_app_sender.send(sub_app).await.unwrap(); + } + }) + .detach(); } +/// This function is used for synchronizing the main app with the render world. +/// Do not call this function if pipelined rendering is not setup. pub fn update_rendering(app_world: &mut World) { - // wait to get the render app back to signal that rendering is finished - let mut render_app = app_world - .resource_scope(|world, main_thread_executor: Mut| { - ComputeTaskPool::get() - .scope(Some(main_thread_executor.0.clone()), |s| { - s.spawn(async { - let receiver = world.get_resource::().unwrap(); - let recv = receiver.0.recv(); - recv.await.unwrap() - }); - }) - .pop() - }) - .unwrap(); + app_world.resource_scope(|world, main_thread_executor: Mut| { + // we use a scope here to run any main thread tasks that the render world still needs to run + // while we wait for the render world to be received. + let mut render_app = ComputeTaskPool::get() + .scope(Some(main_thread_executor.0.clone()), |s| { + s.spawn(async { + let receiver = world.get_resource::().unwrap(); + receiver.0.recv().await.unwrap() + }); + }) + .pop() + .unwrap(); - render_app.extract(app_world); + render_app.extract(world); - app_world.resource_scope(|_world, sender: Mut| { + let sender = world.get_resource::().unwrap(); sender.0.send_blocking(render_app).unwrap(); }); - // frame pacing plugin should run here somehow. i.e. after rendering, but before input handling } diff --git a/crates/bevy_tasks/src/main_thread_executor.rs b/crates/bevy_tasks/src/main_thread_executor.rs index eceb1b5d5c884..bb91faaf91626 100644 --- a/crates/bevy_tasks/src/main_thread_executor.rs +++ b/crates/bevy_tasks/src/main_thread_executor.rs @@ -31,17 +31,17 @@ impl ThreadExecutor { /// Gets the `[MainThreadSpawner]` for the thread executor. /// Use this to spawn tasks that run on the thread this was instatiated on. - pub fn spawner(&self) -> MainThreadSpawner<'static> { - MainThreadSpawner(self.executor.clone()) + pub fn spawner(&self) -> ThreadSpawner<'static> { + ThreadSpawner(self.executor.clone()) } /// Gets the `[MainThreadTicker]` for this executor. /// Use this to tick the executor. /// It only returns the ticker if it's on the thread the executor was created on /// and returns `None` otherwise. - pub fn ticker(&self) -> Option { + pub fn ticker(&self) -> Option { if thread::current().id() == self.thread_id { - return Some(MainThreadTicker { + return Some(ThreadTicker { executor: self.executor.clone(), _marker: PhantomData::default(), }); @@ -50,22 +50,24 @@ impl ThreadExecutor { } } +/// Used to spawn on the [`ThreadExecutor`] #[derive(Debug)] -pub struct MainThreadSpawner<'a>(Arc>); -impl<'a> MainThreadSpawner<'a> { +pub struct ThreadSpawner<'a>(Arc>); +impl<'a> ThreadSpawner<'a> { /// Spawn a task on the main thread pub fn spawn(&self, future: impl Future + Send + 'a) -> Task { self.0.spawn(future) } } +/// Used to tick the [`ThreadExecutor`] #[derive(Debug)] -pub struct MainThreadTicker { +pub struct ThreadTicker { executor: Arc>, // make type not send or sync _marker: PhantomData<*const ()>, } -impl MainThreadTicker { +impl ThreadTicker { /// Tick the main thread executor. /// This needs to be called manually on the thread if it is not being used with /// a `[TaskPool::scope]`. diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 2e74093fd0f6d..9a28037f83789 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -11,7 +11,7 @@ use concurrent_queue::ConcurrentQueue; use futures_lite::{future, FutureExt}; use crate::Task; -use crate::{main_thread_executor::MainThreadSpawner, ThreadExecutor}; +use crate::{main_thread_executor::ThreadSpawner, ThreadExecutor}; /// Used to create a [`TaskPool`] #[derive(Debug, Default, Clone)] @@ -158,7 +158,7 @@ impl TaskPool { /// /// This is similar to `rayon::scope` and `crossbeam::scope` /// - /// The `thread_executor` optional parameter can be used to pass a `[ThreadExecutor]` to + /// The `thread_executor` optional parameter can be used to pass a [`ThreadExecutor`] to /// spawn tasks on when calling `spawn_on_scope`. This can be useful for spawning tasks that /// must run on the main thread. If `None` is passed then `spawn_on_scope` runs tasks on /// the thread `scope` is run on. @@ -258,7 +258,7 @@ impl TaskPool { Arc::new(ThreadExecutor::new()) }; let thread_spawner = thread_executor.spawner(); - let thread_spawner: MainThreadSpawner<'env> = unsafe { mem::transmute(thread_spawner) }; + let thread_spawner: ThreadSpawner<'env> = unsafe { mem::transmute(thread_spawner) }; let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); let spawned_ref: &'env ConcurrentQueue> = unsafe { mem::transmute(&spawned) }; @@ -372,7 +372,7 @@ impl Drop for TaskPool { #[derive(Debug)] pub struct Scope<'scope, 'env: 'scope, T> { executor: &'scope async_executor::Executor<'scope>, - thread_spawner: MainThreadSpawner<'scope>, + thread_spawner: ThreadSpawner<'scope>, spawned: &'scope ConcurrentQueue>, // make `Scope` invariant over 'scope and 'env scope: PhantomData<&'scope mut &'scope ()>, From 5d70b8d1d122c43ebccec93c6fcc6ff6facf18ee Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Wed, 9 Nov 2022 21:21:50 -0800 Subject: [PATCH 16/55] remove executor.run from scope --- crates/bevy_tasks/src/task_pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 9a28037f83789..e4b9338cb1f40 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -295,9 +295,9 @@ impl TaskPool { } }; - executor.run(tick_forever).or(get_results).await + tick_forever.or(get_results).await } else { - executor.run(get_results).await + get_results.await } }) } From dcd2d8371fd07acbd2dbc66e219ecd75f0b8c36f Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Thu, 10 Nov 2022 18:48:54 -0800 Subject: [PATCH 17/55] wrap scope so most uses don't need to pass None --- crates/bevy_ecs/src/query/state.rs | 2 +- .../src/schedule/executor_parallel.rs | 2 +- crates/bevy_gltf/src/loader.rs | 2 +- crates/bevy_render/src/pipelined_rendering.rs | 2 +- crates/bevy_tasks/examples/busy_behavior.rs | 2 +- crates/bevy_tasks/examples/idle_behavior.rs | 2 +- crates/bevy_tasks/src/iter/mod.rs | 32 +++++++-------- crates/bevy_tasks/src/slice.rs | 4 +- crates/bevy_tasks/src/task_pool.rs | 41 +++++++++++-------- 9 files changed, 49 insertions(+), 40 deletions(-) diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index a5d1f3c4f8ffc..b0f263e7fd948 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -990,7 +990,7 @@ impl QueryState { ) { // NOTE: If you are changing query iteration code, remember to update the following places, where relevant: // QueryIter, QueryIterationCursor, QueryManyIter, QueryCombinationIter, QueryState::for_each_unchecked_manual, QueryState::par_for_each_unchecked_manual - ComputeTaskPool::get().scope(None, |scope| { + ComputeTaskPool::get().scope(|scope| { if Q::IS_DENSE && F::IS_DENSE { let tables = &world.storages().tables; for table_id in &self.matched_table_ids { diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index 0ff8dd1fd90e9..eb51487680e82 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -148,7 +148,7 @@ impl ParallelSystemExecutor for ParallelExecutor { .get_resource::() .map(|e| e.0.clone()); - ComputeTaskPool::init(TaskPool::default).scope(thread_executor, |scope| { + ComputeTaskPool::init(TaskPool::default).scope_with_executor(thread_executor, |scope| { self.prepare_systems(scope, systems, world); if self.should_run.count_ones(..) == 0 { return; diff --git a/crates/bevy_gltf/src/loader.rs b/crates/bevy_gltf/src/loader.rs index 8c992df2a47e1..79d19e2898ef4 100644 --- a/crates/bevy_gltf/src/loader.rs +++ b/crates/bevy_gltf/src/loader.rs @@ -409,7 +409,7 @@ async fn load_gltf<'a, 'b>( } else { #[cfg(not(target_arch = "wasm32"))] IoTaskPool::get() - .scope(None, |scope| { + .scope(|scope| { gltf.textures().for_each(|gltf_texture| { let linear_textures = &linear_textures; let load_context: &LoadContext = load_context; diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 0f71b58cb955e..271d68080a231 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -54,7 +54,7 @@ pub fn update_rendering(app_world: &mut World) { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. let mut render_app = ComputeTaskPool::get() - .scope(Some(main_thread_executor.0.clone()), |s| { + .scope_with_executor(Some(main_thread_executor.0.clone()), |s| { s.spawn(async { let receiver = world.get_resource::().unwrap(); receiver.0.recv().await.unwrap() diff --git a/crates/bevy_tasks/examples/busy_behavior.rs b/crates/bevy_tasks/examples/busy_behavior.rs index df78b2316251a..8a74034e0ca90 100644 --- a/crates/bevy_tasks/examples/busy_behavior.rs +++ b/crates/bevy_tasks/examples/busy_behavior.rs @@ -11,7 +11,7 @@ fn main() { .build(); let t0 = instant::Instant::now(); - pool.scope(None, |s| { + pool.scope(|s| { for i in 0..40 { s.spawn(async move { let now = instant::Instant::now(); diff --git a/crates/bevy_tasks/examples/idle_behavior.rs b/crates/bevy_tasks/examples/idle_behavior.rs index b1f5f2adb54b6..daa2eaf2e2a89 100644 --- a/crates/bevy_tasks/examples/idle_behavior.rs +++ b/crates/bevy_tasks/examples/idle_behavior.rs @@ -9,7 +9,7 @@ fn main() { .thread_name("Idle Behavior ThreadPool".to_string()) .build(); - pool.scope(None, |s| { + pool.scope(|s| { for i in 0..1 { s.spawn(async move { println!("Blocking for 10 seconds"); diff --git a/crates/bevy_tasks/src/iter/mod.rs b/crates/bevy_tasks/src/iter/mod.rs index 2c3e2fa673138..952bc53075551 100644 --- a/crates/bevy_tasks/src/iter/mod.rs +++ b/crates/bevy_tasks/src/iter/mod.rs @@ -34,7 +34,7 @@ where /// /// See [`Iterator::count()`](https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.count) fn count(mut self, pool: &TaskPool) -> usize { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.count() }); } @@ -105,7 +105,7 @@ where where F: FnMut(BatchIter::Item) + Send + Clone + Sync, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { @@ -195,7 +195,7 @@ where C: std::iter::FromIterator, BatchIter::Item: Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.collect::>() }); } @@ -216,7 +216,7 @@ where BatchIter::Item: Send + 'static, { let (mut a, mut b) = <(C, C)>::default(); - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.partition::, F>(newf) }); @@ -242,7 +242,7 @@ where F: FnMut(C, BatchIter::Item) -> C + Send + Sync + Clone, C: Clone + Send + Sync + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); let newi = init.clone(); @@ -260,7 +260,7 @@ where where F: FnMut(BatchIter::Item) -> bool + Send + Sync + Clone, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(mut batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.all(newf) }); @@ -279,7 +279,7 @@ where where F: FnMut(BatchIter::Item) -> bool + Send + Sync + Clone, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(mut batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.any(newf) }); @@ -299,7 +299,7 @@ where where F: FnMut(BatchIter::Item) -> bool + Send + Sync + Clone, { - let poses = pool.scope(None, |s| { + let poses = pool.scope(|s| { while let Some(batch) = self.next_batch() { let mut newf = f.clone(); s.spawn(async move { @@ -332,7 +332,7 @@ where where BatchIter::Item: Ord + Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.max() }); } @@ -349,7 +349,7 @@ where where BatchIter::Item: Ord + Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.min() }); } @@ -368,7 +368,7 @@ where F: FnMut(&BatchIter::Item) -> R + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.max_by_key(newf) }); @@ -388,7 +388,7 @@ where F: FnMut(&BatchIter::Item, &BatchIter::Item) -> std::cmp::Ordering + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.max_by(newf) }); @@ -408,7 +408,7 @@ where F: FnMut(&BatchIter::Item) -> R + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.min_by_key(newf) }); @@ -428,7 +428,7 @@ where F: FnMut(&BatchIter::Item, &BatchIter::Item) -> std::cmp::Ordering + Send + Sync + Clone, BatchIter::Item: Send + 'static, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { let newf = f.clone(); s.spawn(async move { batch.min_by(newf) }); @@ -482,7 +482,7 @@ where S: std::iter::Sum + Send + 'static, R: std::iter::Sum, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.sum() }); } @@ -499,7 +499,7 @@ where S: std::iter::Product + Send + 'static, R: std::iter::Product, { - pool.scope(None, |s| { + pool.scope(|s| { while let Some(batch) = self.next_batch() { s.spawn(async move { batch.product() }); } diff --git a/crates/bevy_tasks/src/slice.rs b/crates/bevy_tasks/src/slice.rs index 44372304e71d5..4b5d875ea989b 100644 --- a/crates/bevy_tasks/src/slice.rs +++ b/crates/bevy_tasks/src/slice.rs @@ -37,7 +37,7 @@ pub trait ParallelSlice: AsRef<[T]> { { let slice = self.as_ref(); let f = &f; - task_pool.scope(None, |scope| { + task_pool.scope(|scope| { for chunk in slice.chunks(chunk_size) { scope.spawn(async move { f(chunk) }); } @@ -134,7 +134,7 @@ pub trait ParallelSliceMut: AsMut<[T]> { { let slice = self.as_mut(); let f = &f; - task_pool.scope(None, |scope| { + task_pool.scope(|scope| { for chunk in slice.chunks_mut(chunk_size) { scope.spawn(async move { f(chunk) }); } diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index e4b9338cb1f40..8172c73b2503a 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -158,11 +158,6 @@ impl TaskPool { /// /// This is similar to `rayon::scope` and `crossbeam::scope` /// - /// The `thread_executor` optional parameter can be used to pass a [`ThreadExecutor`] to - /// spawn tasks on when calling `spawn_on_scope`. This can be useful for spawning tasks that - /// must run on the main thread. If `None` is passed then `spawn_on_scope` runs tasks on - /// the thread `scope` is run on. - /// /// # Example /// /// ``` @@ -170,7 +165,7 @@ impl TaskPool { /// /// let pool = TaskPool::new(); /// let mut x = 0; - /// let results = pool.scope(None, |s| { + /// let results = pool.scope(|s| { /// s.spawn(async { /// // you can borrow the spawner inside a task and spawn tasks from within the task /// s.spawn(async { @@ -190,7 +185,7 @@ impl TaskPool { /// assert!(results.contains(&1)); /// /// // The ordering is deterministic if you only spawn directly from the closure function. - /// let results = pool.scope(None, |s| { + /// let results = pool.scope(|s| { /// s.spawn(async { 0 }); /// s.spawn(async { 1 }); /// }); @@ -215,7 +210,7 @@ impl TaskPool { /// fn scope_escapes_closure() { /// let pool = TaskPool::new(); /// let foo = Box::new(42); - /// pool.scope(None, |scope| { + /// pool.scope(|scope| { /// std::thread::spawn(move || { /// // UB. This could spawn on the scope after `.scope` returns and the internal Scope is dropped. /// scope.spawn(async move { @@ -230,7 +225,7 @@ impl TaskPool { /// use bevy_tasks::TaskPool; /// fn cannot_borrow_from_closure() { /// let pool = TaskPool::new(); - /// pool.scope(None, |scope| { + /// pool.scope(|scope| { /// let x = 1; /// let y = &x; /// scope.spawn(async move { @@ -238,8 +233,22 @@ impl TaskPool { /// }); /// }); /// } - /// - pub fn scope<'env, F, T>(&self, thread_executor: Option>, f: F) -> Vec + pub fn scope<'env, F, T>(&self, f: F) -> Vec + where + F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), + T: Send + 'static, + { + self.scope_with_executor(None, f) + } + + /// This allows passing an external executor to spawn tasks on. When you pass an external executor + /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. + /// See [`Self::scope`] for more details in general about how scopes work. + pub fn scope_with_executor<'env, F, T>( + &self, + thread_executor: Option>, + f: F, + ) -> Vec where F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, @@ -440,7 +449,7 @@ mod tests { let count = Arc::new(AtomicI32::new(0)); - let outputs = pool.scope(None, |scope| { + let outputs = pool.scope(|scope| { for _ in 0..100 { let count_clone = count.clone(); scope.spawn(async move { @@ -472,7 +481,7 @@ mod tests { let local_count = Arc::new(AtomicI32::new(0)); let non_local_count = Arc::new(AtomicI32::new(0)); - let outputs = pool.scope(None, |scope| { + let outputs = pool.scope(|scope| { for i in 0..100 { if i % 2 == 0 { let count_clone = non_local_count.clone(); @@ -520,7 +529,7 @@ mod tests { let inner_pool = pool.clone(); let inner_thread_check_failed = thread_check_failed.clone(); std::thread::spawn(move || { - inner_pool.scope(None, |scope| { + inner_pool.scope(|scope| { let inner_count_clone = count_clone.clone(); scope.spawn(async move { inner_count_clone.fetch_add(1, Ordering::Release); @@ -553,7 +562,7 @@ mod tests { let count = Arc::new(AtomicI32::new(0)); - let outputs: Vec = pool.scope(None, |scope| { + let outputs: Vec = pool.scope(|scope| { for _ in 0..10 { let count_clone = count.clone(); scope.spawn(async move { @@ -595,7 +604,7 @@ mod tests { let inner_pool = pool.clone(); let inner_thread_check_failed = thread_check_failed.clone(); std::thread::spawn(move || { - inner_pool.scope(None, |scope| { + inner_pool.scope(|scope| { let spawner = std::thread::current().id(); let inner_count_clone = count_clone.clone(); scope.spawn(async move { From 6e149e4aa655e20e05c293771b4d6007d782a915 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 11:14:23 -0800 Subject: [PATCH 18/55] make a setup function on app to run the setup_rendering function --- crates/bevy_app/src/app.rs | 16 +++++ crates/bevy_render/src/lib.rs | 15 +--- crates/bevy_render/src/pipelined_rendering.rs | 69 ++++++++++++++++--- crates/bevy_winit/Cargo.toml | 1 - crates/bevy_winit/src/lib.rs | 3 - 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 7668786a549e7..a65f7d0ef094f 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -63,6 +63,9 @@ pub struct App { /// The systems of the [`App`] will run using this [`World`]. /// If additional separate [`World`]-[`Schedule`] pairs are needed, you can use [`sub_app`](App::add_sub_app)s. pub world: World, + /// The [setup function](Self::set_setup) is responsible for any final setup needed + /// before calling the runner + setup: Box, /// The [runner function](Self::set_runner) is primarily responsible for managing /// the application's event loop and advancing the [`Schedule`]. /// Typically, it is not configured manually, but set by one of Bevy's built-in plugins. @@ -145,6 +148,7 @@ impl App { Self { world: Default::default(), schedule: Default::default(), + setup: Box::new(empty_setup), runner: Box::new(run_once), sub_apps: HashMap::default(), plugin_registry: Vec::default(), @@ -178,6 +182,8 @@ impl App { let _bevy_app_run_span = info_span!("bevy_app").entered(); let mut app = std::mem::replace(self, App::empty()); + let setup = std::mem::replace(&mut app.setup, Box::new(empty_setup)); + (setup)(&mut app); let runner = std::mem::replace(&mut app.runner, Box::new(run_once)); (runner)(app); } @@ -789,6 +795,14 @@ impl App { self } + /// Sets the function that will be called before the [runner](Self::set_runner) + /// This can be useful when you have work to do before the runner is called, but + /// after plugins have been built + pub fn set_setup(&mut self, setup_fn: impl Fn(&mut App) + 'static + Send + Sync) -> &mut Self { + self.setup = Box::new(setup_fn); + self + } + /// Sets the function that will be called when the app is run. /// /// The runner function `run_fn` is called only once by [`App::run`]. If the @@ -1079,6 +1093,8 @@ fn run_once(mut app: App) { app.update(); } +fn empty_setup(_app: &mut App) {} + /// An event that indicates the [`App`] should exit. This will fully exit the app process at the /// start of the next tick of the schedule. /// diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index e8c5d1142ab10..6266ea422546c 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -38,12 +38,12 @@ pub mod prelude { use globals::GlobalsPlugin; pub use once_cell; -use pipelined_rendering::update_rendering; use prelude::ComputedVisibility; use crate::{ camera::CameraPlugin, mesh::MeshPlugin, + pipelined_rendering::build_pipelined_rendering, render_resource::{PipelineCache, Shader, ShaderLoader}, renderer::{render_system, RenderInstance}, view::{ViewPlugin, WindowRenderPlugin}, @@ -139,10 +139,6 @@ pub mod main_graph { #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] pub struct RenderApp; -/// A Label for the sub app that runs the parts of pipelined rendering that need to run on the main thread. -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] -pub struct PipelinedRenderingApp; - impl Plugin for RenderPlugin { /// Initializes the renderer, sets up the [`RenderStage`](RenderStage) and creates the rendering sub-app. fn build(&self, app: &mut App) { @@ -344,14 +340,7 @@ impl Plugin for RenderPlugin { }); if self.use_pipelined_rendering { - app.add_sub_app( - PipelinedRenderingApp, - App::new(), - |app_world, _render_app| { - update_rendering(app_world); - }, - |_render_world| {}, - ); + build_pipelined_rendering(app); } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 271d68080a231..543894bf37b34 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -1,14 +1,31 @@ use async_channel::{Receiver, Sender}; -use bevy_app::{App, SubApp}; +use bevy_app::{App, AppLabel, SubApp}; use bevy_ecs::{ - schedule::MainThreadExecutor, + schedule::{MainThreadExecutor, Stage, StageLabel, SystemStage}, system::Resource, world::{Mut, World}, }; use bevy_tasks::ComputeTaskPool; -use crate::{PipelinedRenderingApp, RenderApp}; +use crate::RenderApp; + +/// A Label for the sub app that runs the parts of pipelined rendering that need to run on the main thread. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] +pub struct RenderExtractApp; + +/// Labels for stages in the sub app that syncs with the rendering task. +#[derive(Debug, Hash, PartialEq, Eq, Clone, StageLabel)] +pub enum RenderExtractStage { + /// This stage runs after the render schedule starts, but before I/O processing and the main app schedule. + /// This can be useful for something like frame pacing. + /// |-----------------------------------------------------------------| + /// | | BeforeIoAfterRendering | winit events | main schedule | + /// | extract |-------------------------------------------------------| + /// | | rendering schedule | + /// |-----------------------------------------------------------------| + BeforeIoAfterRendering, +} /// Resource for pipelined rendering to send the render app from the main thread to the rendering thread #[derive(Resource)] @@ -18,10 +35,42 @@ pub struct MainToRenderAppSender(pub Sender); #[derive(Resource)] pub struct RenderToMainAppReceiver(pub Receiver); -/// Sets up the render thread and inserts resources into the main app used for controlling the render thread -/// This does nothing if pipelined rendering is not enabled. -pub fn setup_rendering(app: &mut App) { - if app.get_sub_app(PipelinedRenderingApp).is_err() { +pub(crate) fn build_pipelined_rendering(main_app: &mut App) { + let mut app = App::new(); + main_app.set_setup(setup_rendering); + main_app.add_stage( + RenderExtractStage::BeforeIoAfterRendering, + SystemStage::parallel(), + ); + app.add_sub_app( + RenderExtractApp, + App::new(), + |app_world, _render_app| { + update_rendering(app_world); + }, + |render_app| { + { + #[cfg(feature = "trace")] + let _stage_span = + bevy_utils::tracing::info_span!("stage", name = "before_io_after_rendering") + .entered(); + + // render + let render = render_app + .schedule + .get_stage_mut::(RenderExtractStage::BeforeIoAfterRendering) + .unwrap(); + render.run(&mut render_app.world); + } + }, + ); +} + +// Sets up the render thread and inserts resources into the main app used for controlling the render thread. +// This should be called after plugins have all been built as it removes the rendering sub app from the main app. +// This does nothing if pipelined rendering is not enabled. +fn setup_rendering(app: &mut App) { + if app.get_sub_app(RenderExtractApp).is_err() { return; } @@ -47,9 +96,9 @@ pub fn setup_rendering(app: &mut App) { .detach(); } -/// This function is used for synchronizing the main app with the render world. -/// Do not call this function if pipelined rendering is not setup. -pub fn update_rendering(app_world: &mut World) { +// This function is used for synchronizing the main app with the render world. +// Do not call this function if pipelined rendering is not setup. +fn update_rendering(app_world: &mut World) { app_world.resource_scope(|world, main_thread_executor: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. diff --git a/crates/bevy_winit/Cargo.toml b/crates/bevy_winit/Cargo.toml index d9100767c88b1..96864623f1113 100644 --- a/crates/bevy_winit/Cargo.toml +++ b/crates/bevy_winit/Cargo.toml @@ -15,7 +15,6 @@ x11 = ["winit/x11"] [dependencies] # bevy -bevy_render = { path = "../bevy_render", version = "0.9.0-dev" } bevy_app = { path = "../bevy_app", version = "0.9.0" } bevy_ecs = { path = "../bevy_ecs", version = "0.9.0" } bevy_input = { path = "../bevy_input", version = "0.9.0" } diff --git a/crates/bevy_winit/src/lib.rs b/crates/bevy_winit/src/lib.rs index 32b301f7f2124..749d256882500 100644 --- a/crates/bevy_winit/src/lib.rs +++ b/crates/bevy_winit/src/lib.rs @@ -4,7 +4,6 @@ mod web_resize; mod winit_config; mod winit_windows; -use bevy_render::pipelined_rendering::setup_rendering; use winit::window::CursorGrabMode; pub use winit_config::*; pub use winit_windows::*; @@ -369,8 +368,6 @@ pub fn winit_runner_with(mut app: App) { let return_from_run = app.world.resource::().return_from_run; - setup_rendering(&mut app); - trace!("Entering winit event loop"); let event_handler = move |event: Event<()>, From a53cb13e76ac6eeaa8879c39c0462cbe89ea12e6 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 11:50:42 -0800 Subject: [PATCH 19/55] change pipelined rendering into a plugin --- crates/bevy_internal/src/default_plugins.rs | 6 ++ crates/bevy_render/src/lib.rs | 26 ++------ crates/bevy_render/src/pipelined_rendering.rs | 66 ++++++++++--------- 3 files changed, 45 insertions(+), 53 deletions(-) diff --git a/crates/bevy_internal/src/default_plugins.rs b/crates/bevy_internal/src/default_plugins.rs index 68dbc7ac89fba..f35d64d204477 100644 --- a/crates/bevy_internal/src/default_plugins.rs +++ b/crates/bevy_internal/src/default_plugins.rs @@ -64,6 +64,12 @@ impl PluginGroup for DefaultPlugins { // NOTE: Load this after renderer initialization so that it knows about the supported // compressed texture formats .add(bevy_render::texture::ImagePlugin::default()); + + #[cfg(not(target_arch = "wasm"))] + { + group = group + .add(bevy_render::pipelined_rendering::PipelinedRenderingPlugin::default()); + } } #[cfg(feature = "bevy_core_pipeline")] diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 6266ea422546c..b3954de5d0718 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -43,7 +43,8 @@ use prelude::ComputedVisibility; use crate::{ camera::CameraPlugin, mesh::MeshPlugin, - pipelined_rendering::build_pipelined_rendering, + primitives::{CubemapFrusta, Frustum}, + render_graph::RenderGraph, render_resource::{PipelineCache, Shader, ShaderLoader}, renderer::{render_system, RenderInstance}, view::{ViewPlugin, WindowRenderPlugin}, @@ -58,23 +59,8 @@ use std::{ }; /// Contains the default Bevy rendering backend based on wgpu. -pub struct RenderPlugin { - /// Pipelined rendering runs the rendering simultaneously with the main app. - /// Use this to turn pipelined rendering on or off. By default it's on in native - /// environments and off in wasm. - pub use_pipelined_rendering: bool, -} - -impl Default for RenderPlugin { - fn default() -> Self { - RenderPlugin { - #[cfg(not(target_arch = "wasm32"))] - use_pipelined_rendering: true, - #[cfg(target_arch = "wasm32")] - use_pipelined_rendering: false, - } - } -} +#[derive(Default)] +pub struct RenderPlugin; /// The labels of the default App rendering stages. #[derive(Debug, Hash, PartialEq, Eq, Clone, StageLabel)] @@ -338,10 +324,6 @@ impl Plugin for RenderPlugin { render_app.world.clear_entities(); } }); - - if self.use_pipelined_rendering { - build_pipelined_rendering(app); - } } app.add_plugin(ValidParentCheckPlugin::::default()) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 543894bf37b34..cc163ff87f98a 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -1,6 +1,6 @@ use async_channel::{Receiver, Sender}; -use bevy_app::{App, AppLabel, SubApp}; +use bevy_app::{App, AppLabel, Plugin, SubApp}; use bevy_ecs::{ schedule::{MainThreadExecutor, Stage, StageLabel, SystemStage}, system::Resource, @@ -35,35 +35,39 @@ pub struct MainToRenderAppSender(pub Sender); #[derive(Resource)] pub struct RenderToMainAppReceiver(pub Receiver); -pub(crate) fn build_pipelined_rendering(main_app: &mut App) { - let mut app = App::new(); - main_app.set_setup(setup_rendering); - main_app.add_stage( - RenderExtractStage::BeforeIoAfterRendering, - SystemStage::parallel(), - ); - app.add_sub_app( - RenderExtractApp, - App::new(), - |app_world, _render_app| { - update_rendering(app_world); - }, - |render_app| { - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "before_io_after_rendering") - .entered(); - - // render - let render = render_app - .schedule - .get_stage_mut::(RenderExtractStage::BeforeIoAfterRendering) - .unwrap(); - render.run(&mut render_app.world); - } - }, - ); +#[derive(Default)] +pub struct PipelinedRenderingPlugin; +impl Plugin for PipelinedRenderingPlugin { + fn build(&self, app: &mut App) { + let mut sub_app = App::new(); + app.set_setup(setup_rendering); + app.add_stage( + RenderExtractStage::BeforeIoAfterRendering, + SystemStage::parallel(), + ); + sub_app.add_sub_app( + RenderExtractApp, + App::new(), + update_rendering, + |render_app| { + { + #[cfg(feature = "trace")] + let _stage_span = bevy_utils::tracing::info_span!( + "stage", + name = "before_io_after_rendering" + ) + .entered(); + + // render + let render = render_app + .schedule + .get_stage_mut::(RenderExtractStage::BeforeIoAfterRendering) + .unwrap(); + render.run(&mut render_app.world); + } + }, + ); + } } // Sets up the render thread and inserts resources into the main app used for controlling the render thread. @@ -98,7 +102,7 @@ fn setup_rendering(app: &mut App) { // This function is used for synchronizing the main app with the render world. // Do not call this function if pipelined rendering is not setup. -fn update_rendering(app_world: &mut World) { +fn update_rendering(app_world: &mut World, _sub_app: &mut App) { app_world.resource_scope(|world, main_thread_executor: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. From 3cd2122aef43a3d9c2cb694e1fa0020dd1f9b40c Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 13:00:30 -0800 Subject: [PATCH 20/55] fix pipelined rendering --- crates/bevy_app/src/app.rs | 8 ++-- crates/bevy_render/src/pipelined_rendering.rs | 44 ++++++------------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index a65f7d0ef094f..1e8320283c6a6 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -162,9 +162,11 @@ impl App { /// /// See [`add_sub_app`](Self::add_sub_app) and [`run_once`](Schedule::run_once) for more details. pub fn update(&mut self) { - #[cfg(feature = "trace")] - let _bevy_frame_update_span = info_span!("main app").entered(); - self.schedule.run(&mut self.world); + { + #[cfg(feature = "trace")] + let _bevy_frame_update_span = info_span!("main app").entered(); + self.schedule.run(&mut self.world); + } for sub_app in self.sub_apps.values_mut() { sub_app.extract(&mut self.world); sub_app.run(); diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index cc163ff87f98a..7d9abf37be69d 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -2,7 +2,7 @@ use async_channel::{Receiver, Sender}; use bevy_app::{App, AppLabel, Plugin, SubApp}; use bevy_ecs::{ - schedule::{MainThreadExecutor, Stage, StageLabel, SystemStage}, + schedule::{MainThreadExecutor, StageLabel, SystemStage}, system::Resource, world::{Mut, World}, }; @@ -19,12 +19,12 @@ pub struct RenderExtractApp; pub enum RenderExtractStage { /// This stage runs after the render schedule starts, but before I/O processing and the main app schedule. /// This can be useful for something like frame pacing. - /// |-----------------------------------------------------------------| - /// | | BeforeIoAfterRendering | winit events | main schedule | - /// | extract |-------------------------------------------------------| - /// | | rendering schedule | - /// |-----------------------------------------------------------------| - BeforeIoAfterRendering, + /// |-------------------------------------------------------------------| + /// | | BeforeIoAfterRenderStart | winit events | main schedule | + /// | extract |---------------------------------------------------------| + /// | | rendering schedule | + /// |-------------------------------------------------------------------| + BeforeIoAfterRenderStart, } /// Resource for pipelined rendering to send the render app from the main thread to the rendering thread @@ -41,32 +41,13 @@ impl Plugin for PipelinedRenderingPlugin { fn build(&self, app: &mut App) { let mut sub_app = App::new(); app.set_setup(setup_rendering); - app.add_stage( - RenderExtractStage::BeforeIoAfterRendering, + sub_app.add_stage( + RenderExtractStage::BeforeIoAfterRenderStart, SystemStage::parallel(), ); - sub_app.add_sub_app( - RenderExtractApp, - App::new(), - update_rendering, - |render_app| { - { - #[cfg(feature = "trace")] - let _stage_span = bevy_utils::tracing::info_span!( - "stage", - name = "before_io_after_rendering" - ) - .entered(); - - // render - let render = render_app - .schedule - .get_stage_mut::(RenderExtractStage::BeforeIoAfterRendering) - .unwrap(); - render.run(&mut render_app.world); - } - }, - ); + app.add_sub_app(RenderExtractApp, sub_app, update_rendering, |render_app| { + render_app.run(); + }); } } @@ -74,6 +55,7 @@ impl Plugin for PipelinedRenderingPlugin { // This should be called after plugins have all been built as it removes the rendering sub app from the main app. // This does nothing if pipelined rendering is not enabled. fn setup_rendering(app: &mut App) { + // skip setting up when headless if app.get_sub_app(RenderExtractApp).is_err() { return; } From 799511f6bfc27e48e0998416ca63c1189e90c2ec Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 13:26:07 -0800 Subject: [PATCH 21/55] fix wasm again --- .../src/single_threaded_task_pool.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index cdcfdaa7c05c7..61fd75a3472d2 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -72,7 +72,24 @@ impl TaskPool { /// to spawn tasks. This function will await the completion of all tasks before returning. /// /// This is similar to `rayon::scope` and `crossbeam::scope` - pub fn scope<'env, F, T>(&self, _thread_executor: Option>, f: F) -> Vec + pub fn scope<'env, F, T>(&self, f: F) -> Vec + where + F: for<'scope> FnOnce(&'env mut Scope<'scope, 'env, T>), + T: Send + 'static, + { + self.scope_with_executor(None, f) + } + + /// Allows spawning non-`static futures on the thread pool. The function takes a callback, + /// passing a scope object into it. The scope object provided to the callback can be used + /// to spawn tasks. This function will await the completion of all tasks before returning. + /// + /// This is similar to `rayon::scope` and `crossbeam::scope` + pub fn scope_with_executor<'env, F, T>( + &self, + _thread_executor: Option>, + f: F, + ) -> Vec where F: for<'scope> FnOnce(&'env mut Scope<'scope, 'env, T>), T: Send + 'static, From d306874d0855a2d7b1876d2ce33463470e0505a4 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 14:28:18 -0800 Subject: [PATCH 22/55] move setup to plugin --- crates/bevy_app/src/app.rs | 24 +++------ crates/bevy_app/src/plugin.rs | 6 +++ crates/bevy_render/src/pipelined_rendering.rs | 50 +++++++++---------- 3 files changed, 37 insertions(+), 43 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 1e8320283c6a6..fde6fe9ea2c5f 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -63,9 +63,6 @@ pub struct App { /// The systems of the [`App`] will run using this [`World`]. /// If additional separate [`World`]-[`Schedule`] pairs are needed, you can use [`sub_app`](App::add_sub_app)s. pub world: World, - /// The [setup function](Self::set_setup) is responsible for any final setup needed - /// before calling the runner - setup: Box, /// The [runner function](Self::set_runner) is primarily responsible for managing /// the application's event loop and advancing the [`Schedule`]. /// Typically, it is not configured manually, but set by one of Bevy's built-in plugins. @@ -148,7 +145,6 @@ impl App { Self { world: Default::default(), schedule: Default::default(), - setup: Box::new(empty_setup), runner: Box::new(run_once), sub_apps: HashMap::default(), plugin_registry: Vec::default(), @@ -184,8 +180,14 @@ impl App { let _bevy_app_run_span = info_span!("bevy_app").entered(); let mut app = std::mem::replace(self, App::empty()); - let setup = std::mem::replace(&mut app.setup, Box::new(empty_setup)); - (setup)(&mut app); + + // temporarily remove the plugin registry to run each plugin's setup function on app. + let mut plugin_registry = std::mem::take(&mut app.plugin_registry); + for plugin in &plugin_registry { + plugin.setup(&mut app); + } + std::mem::swap(&mut app.plugin_registry, &mut plugin_registry); + let runner = std::mem::replace(&mut app.runner, Box::new(run_once)); (runner)(app); } @@ -797,14 +799,6 @@ impl App { self } - /// Sets the function that will be called before the [runner](Self::set_runner) - /// This can be useful when you have work to do before the runner is called, but - /// after plugins have been built - pub fn set_setup(&mut self, setup_fn: impl Fn(&mut App) + 'static + Send + Sync) -> &mut Self { - self.setup = Box::new(setup_fn); - self - } - /// Sets the function that will be called when the app is run. /// /// The runner function `run_fn` is called only once by [`App::run`]. If the @@ -1095,8 +1089,6 @@ fn run_once(mut app: App) { app.update(); } -fn empty_setup(_app: &mut App) {} - /// An event that indicates the [`App`] should exit. This will fully exit the app process at the /// start of the next tick of the schedule. /// diff --git a/crates/bevy_app/src/plugin.rs b/crates/bevy_app/src/plugin.rs index 7e6e0c575a4d7..afcf90d82934f 100644 --- a/crates/bevy_app/src/plugin.rs +++ b/crates/bevy_app/src/plugin.rs @@ -16,6 +16,12 @@ use std::any::Any; pub trait Plugin: Downcast + Any + Send + Sync { /// Configures the [`App`] to which this plugin is added. fn build(&self, app: &mut App); + /// Runs after all plugins are built, but before the app runner is called. + /// This can be useful if you have some resource that other plugins need during their build step, + /// but after build you want to remove it and send it to another thread. + fn setup(&self, _app: &mut App) { + // do nothing + } /// Configures a name for the [`Plugin`] which is primarily used for debugging. fn name(&self) -> &str { std::any::type_name::() diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 7d9abf37be69d..9fc84aec546cc 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -40,7 +40,6 @@ pub struct PipelinedRenderingPlugin; impl Plugin for PipelinedRenderingPlugin { fn build(&self, app: &mut App) { let mut sub_app = App::new(); - app.set_setup(setup_rendering); sub_app.add_stage( RenderExtractStage::BeforeIoAfterRenderStart, SystemStage::parallel(), @@ -49,37 +48,34 @@ impl Plugin for PipelinedRenderingPlugin { render_app.run(); }); } -} -// Sets up the render thread and inserts resources into the main app used for controlling the render thread. -// This should be called after plugins have all been built as it removes the rendering sub app from the main app. -// This does nothing if pipelined rendering is not enabled. -fn setup_rendering(app: &mut App) { - // skip setting up when headless - if app.get_sub_app(RenderExtractApp).is_err() { - return; - } + // Sets up the render thread and inserts resources into the main app used for controlling the render thread. + fn setup(&self, app: &mut App) { + // skip setting up when headless + if app.get_sub_app(RenderExtractApp).is_err() { + return; + } - let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::(1); - let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::(1); + let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::(1); + let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::(1); - let render_app = app.remove_sub_app(RenderApp).unwrap(); - render_to_app_sender.send_blocking(render_app).unwrap(); + let render_app = app.remove_sub_app(RenderApp).unwrap(); + render_to_app_sender.send_blocking(render_app).unwrap(); - app.insert_resource(MainToRenderAppSender(app_to_render_sender)); - app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver)); + app.insert_resource(MainToRenderAppSender(app_to_render_sender)); + app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver)); - ComputeTaskPool::get() - .spawn(async move { - loop { - // TODO: exit loop when app is exited - let recv_task = app_to_render_receiver.recv(); - let mut sub_app = recv_task.await.unwrap(); - sub_app.run(); - render_to_app_sender.send(sub_app).await.unwrap(); - } - }) - .detach(); + ComputeTaskPool::get() + .spawn(async move { + loop { + let recv_task = app_to_render_receiver.recv(); + let mut sub_app = recv_task.await.unwrap(); + sub_app.run(); + render_to_app_sender.send(sub_app).await.unwrap(); + } + }) + .detach(); + } } // This function is used for synchronizing the main app with the render world. From f2507fa3c1012ff3baa7f7165b9b82eda34a377f Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 15:23:03 -0800 Subject: [PATCH 23/55] move cloning the MainThreadExecutor to setup --- crates/bevy_app/src/app.rs | 15 ++++++++------- crates/bevy_render/src/pipelined_rendering.rs | 7 ++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index fde6fe9ea2c5f..6602cda429ae2 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -4,8 +4,8 @@ use bevy_ecs::{ event::{Event, Events}, prelude::FromWorld, schedule::{ - IntoSystemDescriptor, MainThreadExecutor, Schedule, ShouldRun, Stage, StageLabel, State, - StateData, SystemSet, SystemStage, + IntoSystemDescriptor, Schedule, ShouldRun, Stage, StageLabel, State, StateData, SystemSet, + SystemStage, }, system::Resource, world::World, @@ -87,7 +87,8 @@ impl Debug for App { /// Each `SubApp` has its own [`Schedule`] and [`World`], enabling a separation of concerns. pub struct SubApp { - app: App, + /// The [`SubApp`]'s instance of [`App`] + pub app: App, extract: Box, // Send + Sync bound is only required to make SubApp send sync runner: Box, // this Send + Sync bound is required since we're running this function on another thread } @@ -1013,13 +1014,13 @@ impl App { pub fn add_sub_app( &mut self, label: impl AppLabel, - mut app: App, + app: App, extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, runner: impl Fn(&mut App) + 'static + Send + Sync, ) -> &mut Self { - if let Some(executor) = self.world.get_resource::() { - app.world.insert_resource(executor.clone()); - } + // if let Some(executor) = self.world.get_resource::() { + // app.world.insert_resource(executor.clone()); + // } self.sub_apps.insert( label.as_label(), SubApp { diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 9fc84aec546cc..d008b25652cca 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -59,7 +59,12 @@ impl Plugin for PipelinedRenderingPlugin { let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::(1); let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::(1); - let render_app = app.remove_sub_app(RenderApp).unwrap(); + let mut render_app = app.remove_sub_app(RenderApp).unwrap(); + + // clone main thread executor to render world + let executor = app.world.get_resource::().unwrap(); + render_app.app.world.insert_resource(executor.clone()); + render_to_app_sender.send_blocking(render_app).unwrap(); app.insert_resource(MainToRenderAppSender(app_to_render_sender)); From 8f5f259e1a9d415a3cdf9ca8bc06bda70cdca320 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Fri, 11 Nov 2022 18:05:46 -0800 Subject: [PATCH 24/55] remove runner and just run the schedule --- crates/bevy_app/src/app.rs | 8 +- crates/bevy_ecs/src/schedule/mod.rs | 10 ++ crates/bevy_render/src/lib.rs | 113 +++++------------- crates/bevy_render/src/pipelined_rendering.rs | 4 +- 4 files changed, 43 insertions(+), 92 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 6602cda429ae2..4238fd3ca68a1 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -90,13 +90,12 @@ pub struct SubApp { /// The [`SubApp`]'s instance of [`App`] pub app: App, extract: Box, // Send + Sync bound is only required to make SubApp send sync - runner: Box, // this Send + Sync bound is required since we're running this function on another thread } impl SubApp { /// runs the `SubApp` with its runner pub fn run(&mut self) { - (self.runner)(&mut self.app); + self.app.schedule.run(&mut self.app.world); } /// extract data from main world to sub app @@ -1016,17 +1015,12 @@ impl App { label: impl AppLabel, app: App, extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, - runner: impl Fn(&mut App) + 'static + Send + Sync, ) -> &mut Self { - // if let Some(executor) = self.world.get_resource::() { - // app.world.insert_resource(executor.clone()); - // } self.sub_apps.insert( label.as_label(), SubApp { app, extract: Box::new(extract), - runner: Box::new(runner), }, ); self diff --git a/crates/bevy_ecs/src/schedule/mod.rs b/crates/bevy_ecs/src/schedule/mod.rs index 222339606fec3..346afa585fc0b 100644 --- a/crates/bevy_ecs/src/schedule/mod.rs +++ b/crates/bevy_ecs/src/schedule/mod.rs @@ -361,6 +361,16 @@ impl Schedule { .and_then(|stage| stage.downcast_mut::()) } + pub fn remove_stage(&mut self, stage_label: impl StageLabel) -> Option> { + let label = stage_label.as_label(); + + let Some(index) = self.stage_order.iter().position(|x| *x == label) else { + return None; + }; + self.stage_order.remove(index); + self.stages.remove(&label) + } + /// Executes each [`Stage`] contained in the schedule, one at a time. pub fn run_once(&mut self, world: &mut World) { for label in &self.stage_order { diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index b3954de5d0718..a00856a1d301a 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -90,6 +90,10 @@ pub enum RenderStage { Cleanup, } +/// Resource for holding the extract stage of the rendering schedule +#[derive(Resource)] +pub struct ExtractStage(pub SystemStage); + /// The simulation [`World`] of the application, stored as a resource. /// This resource is only available during [`RenderStage::Extract`] and not /// during command application of that stage. @@ -188,6 +192,9 @@ impl Plugin for RenderPlugin { // after access to the main world is removed // See also https://github.com/bevyengine/bevy/issues/5082 extract_stage.set_apply_buffers(false); + fn clear_entities(world: &mut World) { + world.clear_entities(); + } render_app .add_stage(RenderStage::Extract, extract_stage) .add_stage(RenderStage::Prepare, SystemStage::parallel()) @@ -199,8 +206,11 @@ impl Plugin for RenderPlugin { .with_system(PipelineCache::process_pipeline_queue_system) .with_system(render_system.at_end()), ) - .add_stage(RenderStage::Cleanup, SystemStage::parallel()) - .init_resource::() + .add_stage( + RenderStage::Cleanup, + SystemStage::parallel().with_system(clear_entities.at_end()), + ) + .init_resource::() .insert_resource(RenderInstance(instance)) .insert_resource(device) .insert_resource(queue) @@ -249,80 +259,6 @@ impl Plugin for RenderPlugin { // extract extract(app_world, render_app); } - }, |render_app| { - #[cfg(feature = "trace")] - let _render_span = bevy_utils::tracing::info_span!("render app").entered(); - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "prepare").entered(); - - // prepare - let prepare = render_app - .schedule - .get_stage_mut::(RenderStage::Prepare) - .unwrap(); - prepare.run(&mut render_app.world); - } - - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "queue").entered(); - - // queue - let queue = render_app - .schedule - .get_stage_mut::(RenderStage::Queue) - .unwrap(); - queue.run(&mut render_app.world); - } - - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "sort").entered(); - - // phase sort - let phase_sort = render_app - .schedule - .get_stage_mut::(RenderStage::PhaseSort) - .unwrap(); - phase_sort.run(&mut render_app.world); - } - - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "render").entered(); - - // render - let render = render_app - .schedule - .get_stage_mut::(RenderStage::Render) - .unwrap(); - render.run(&mut render_app.world); - } - - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "cleanup").entered(); - - // cleanup - let cleanup = render_app - .schedule - .get_stage_mut::(RenderStage::Cleanup) - .unwrap(); - cleanup.run(&mut render_app.world); - } - { - #[cfg(feature = "trace")] - let _stage_span = - bevy_utils::tracing::info_span!("stage", name = "clear_entities").entered(); - - render_app.world.clear_entities(); - } }); } @@ -338,6 +274,20 @@ impl Plugin for RenderPlugin { .register_type::() .register_type::(); } + + fn setup(&self, app: &mut App) { + // move stage to resource so render_app.run() doesn't run it. + let render_app = app.get_sub_app_mut(RenderApp).unwrap(); + + let stage = render_app + .schedule + .remove_stage(RenderStage::Extract) + .unwrap() + .downcast::() + .unwrap(); + + render_app.world.insert_resource(ExtractStage(*stage)); + } } /// A "scratch" world used to avoid allocating new worlds every frame when @@ -348,10 +298,7 @@ struct ScratchMainWorld(World); /// Executes the [`Extract`](RenderStage::Extract) stage of the renderer. /// This updates the render world with the extracted ECS data of the current frame. fn extract(app_world: &mut World, render_app: &mut App) { - let extract = render_app - .schedule - .get_stage_mut::(RenderStage::Extract) - .unwrap(); + let mut extract = render_app.world.remove_resource::().unwrap(); // temporarily add the app world to the render world as a resource let scratch_world = app_world.remove_resource::().unwrap(); @@ -359,7 +306,7 @@ fn extract(app_world: &mut World, render_app: &mut App) { let running_world = &mut render_app.world; running_world.insert_resource(MainWorld(inserted_world)); - extract.run(running_world); + extract.0.run(running_world); // move the app world back, as if nothing happened. let inserted_world = running_world.remove_resource::().unwrap(); let scratch_world = std::mem::replace(app_world, inserted_world.0); @@ -368,5 +315,7 @@ fn extract(app_world: &mut World, render_app: &mut App) { // Note: We apply buffers (read, Commands) after the `MainWorld` has been removed from the render app's world // so that in future, pipelining will be able to do this too without any code relying on it. // see - extract.apply_buffers(running_world); + extract.0.apply_buffers(running_world); + + render_app.world.insert_resource(extract); } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index d008b25652cca..c7d4646c9ecb3 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -44,9 +44,7 @@ impl Plugin for PipelinedRenderingPlugin { RenderExtractStage::BeforeIoAfterRenderStart, SystemStage::parallel(), ); - app.add_sub_app(RenderExtractApp, sub_app, update_rendering, |render_app| { - render_app.run(); - }); + app.add_sub_app(RenderExtractApp, sub_app, update_rendering); } // Sets up the render thread and inserts resources into the main app used for controlling the render thread. From 1bf771cdb3cd4ff9258708de6fce975ce430fb19 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Sun, 13 Nov 2022 14:20:16 -0800 Subject: [PATCH 25/55] fix headless example --- crates/bevy_render/src/lib.rs | 22 +++++++++---------- crates/bevy_render/src/pipelined_rendering.rs | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index a00856a1d301a..586e30ef4fab8 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -276,17 +276,17 @@ impl Plugin for RenderPlugin { } fn setup(&self, app: &mut App) { - // move stage to resource so render_app.run() doesn't run it. - let render_app = app.get_sub_app_mut(RenderApp).unwrap(); - - let stage = render_app - .schedule - .remove_stage(RenderStage::Extract) - .unwrap() - .downcast::() - .unwrap(); - - render_app.world.insert_resource(ExtractStage(*stage)); + if let Ok(render_app) = app.get_sub_app_mut(RenderApp) { + // move stage to resource so render_app.run() doesn't run it. + let stage = render_app + .schedule + .remove_stage(RenderStage::Extract) + .unwrap() + .downcast::() + .unwrap(); + + render_app.world.insert_resource(ExtractStage(*stage)); + } } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index c7d4646c9ecb3..ad1d4db6b09c9 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -39,6 +39,10 @@ pub struct RenderToMainAppReceiver(pub Receiver); pub struct PipelinedRenderingPlugin; impl Plugin for PipelinedRenderingPlugin { fn build(&self, app: &mut App) { + // Don't add RenderExtractApp if RenderApp isn't initialized. + if app.get_sub_app(RenderApp).is_err() { + return; + } let mut sub_app = App::new(); sub_app.add_stage( RenderExtractStage::BeforeIoAfterRenderStart, From c537afb2a94aaddbe4b66ababf5a2af19098692a Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Sun, 13 Nov 2022 15:58:07 -0800 Subject: [PATCH 26/55] cleanup --- crates/bevy_app/src/app.rs | 3 ++- crates/bevy_core/src/task_pool_options.rs | 4 ++-- crates/bevy_ecs/src/schedule/mod.rs | 1 + crates/bevy_render/src/pipelined_rendering.rs | 8 ++++---- crates/bevy_tasks/src/task_pool.rs | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 4238fd3ca68a1..face53c99675e 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -1007,7 +1007,8 @@ impl App { /// Adds an [`App`] as a child of the current one. /// - /// The provided functions `extract` and `runner` are normally called by the [`update`](Self::update) method. The [`World`] + /// The provided function `extract` is normally called by the [`update`](Self::update) method. + /// After extract is called, the [`Schedule`] of the sub app is run. The [`World`] /// parameter represents the main app world, while the [`App`] parameter is just a mutable /// reference to the `SubApp` itself. pub fn add_sub_app( diff --git a/crates/bevy_core/src/task_pool_options.rs b/crates/bevy_core/src/task_pool_options.rs index 4537354a69c05..05624c61fdc30 100644 --- a/crates/bevy_core/src/task_pool_options.rs +++ b/crates/bevy_core/src/task_pool_options.rs @@ -57,14 +57,14 @@ impl Default for TaskPoolOptions { min_total_threads: 1, max_total_threads: std::usize::MAX, - // Use 25% of cores for IO, at least 1, no more than 4 + // Use 10% of cores for IO, at least 1, no more than 4 io: TaskPoolThreadAssignmentPolicy { min_threads: 1, max_threads: 4, percent: 0.25, }, - // Use 25% of cores for async compute, at least 1, no more than 4 + // Use 10% of cores for async compute, at least 1, no more than 4 async_compute: TaskPoolThreadAssignmentPolicy { min_threads: 1, max_threads: 4, diff --git a/crates/bevy_ecs/src/schedule/mod.rs b/crates/bevy_ecs/src/schedule/mod.rs index 346afa585fc0b..b5fe8731fb452 100644 --- a/crates/bevy_ecs/src/schedule/mod.rs +++ b/crates/bevy_ecs/src/schedule/mod.rs @@ -361,6 +361,7 @@ impl Schedule { .and_then(|stage| stage.downcast_mut::()) } + /// Remove a [`Stage`] from the schedule pub fn remove_stage(&mut self, stage_label: impl StageLabel) -> Option> { let label = stage_label.as_label(); diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index ad1d4db6b09c9..7b620679dd6f1 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -27,11 +27,11 @@ pub enum RenderExtractStage { BeforeIoAfterRenderStart, } -/// Resource for pipelined rendering to send the render app from the main thread to the rendering thread +/// Channel to send the render app from the main thread to the rendering thread #[derive(Resource)] pub struct MainToRenderAppSender(pub Sender); -/// Resource for pipelined rendering to send the render app from the render thread to the main thread +/// Channel to send the render app from the render thread to the main thread #[derive(Resource)] pub struct RenderToMainAppReceiver(pub Receiver); @@ -85,8 +85,8 @@ impl Plugin for PipelinedRenderingPlugin { } } -// This function is used for synchronizing the main app with the render world. -// Do not call this function if pipelined rendering is not setup. +// This function is waits for the rendering world to be sent back, +// runs extract, and then sends the rendering world back to the render thread. fn update_rendering(app_world: &mut World, _sub_app: &mut App) { app_world.resource_scope(|world, main_thread_executor: Mut| { // we use a scope here to run any main thread tasks that the render world still needs to run diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 8172c73b2503a..2dd36877a1776 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -472,7 +472,7 @@ mod tests { } #[test] - fn test_mixed_spawn_on_main_and_spawn() { + fn test_mixed_spawn_on_scope_and_spawn() { let pool = TaskPool::new(); let foo = Box::new(42); From 98e9c0410a5f2ee334fb456d127b916e1fd8342c Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 14 Nov 2022 19:47:22 -0800 Subject: [PATCH 27/55] add render app span --- crates/bevy_render/src/pipelined_rendering.rs | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 7b620679dd6f1..7a3fcd3aa9508 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -7,6 +7,8 @@ use bevy_ecs::{ world::{Mut, World}, }; use bevy_tasks::ComputeTaskPool; +#[cfg(feature = "trace")] +use bevy_utils::tracing::Instrument; use crate::RenderApp; @@ -72,16 +74,20 @@ impl Plugin for PipelinedRenderingPlugin { app.insert_resource(MainToRenderAppSender(app_to_render_sender)); app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver)); - ComputeTaskPool::get() - .spawn(async move { - loop { - let recv_task = app_to_render_receiver.recv(); - let mut sub_app = recv_task.await.unwrap(); - sub_app.run(); - render_to_app_sender.send(sub_app).await.unwrap(); - } - }) - .detach(); + let render_task = async move { + loop { + let recv_task = app_to_render_receiver.recv(); + let mut sub_app = recv_task.await.unwrap(); + sub_app.run(); + render_to_app_sender.send(sub_app).await.unwrap(); + } + }; + #[cfg(feature = "trace")] + let span = bevy_utils::tracing::info_span!("render app"); + #[cfg(feature = "trace")] + let render_task = render_task.instrument(span); + + ComputeTaskPool::get().spawn(render_task).detach(); } } From 24241a4ad46180528e2323798cfdd8546c4f3019 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 14 Nov 2022 20:47:38 -0800 Subject: [PATCH 28/55] move inserting MainThreadExecutor to PipelinedRenderingPlugin --- crates/bevy_core/src/lib.rs | 2 -- crates/bevy_render/src/pipelined_rendering.rs | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/bevy_core/src/lib.rs b/crates/bevy_core/src/lib.rs index 4b64f7efcc0e6..ee21feb2ec9ef 100644 --- a/crates/bevy_core/src/lib.rs +++ b/crates/bevy_core/src/lib.rs @@ -6,7 +6,6 @@ mod name; mod serde; mod task_pool_options; -use bevy_ecs::schedule::MainThreadExecutor; use bevy_ecs::system::{ResMut, Resource}; pub use bytemuck::{bytes_of, cast_slice, Pod, Zeroable}; pub use name::*; @@ -43,7 +42,6 @@ impl Plugin for CorePlugin { fn build(&self, app: &mut App) { // Setup the default bevy task pools self.task_pool_options.create_default_pools(); - app.insert_resource(MainThreadExecutor::new()); #[cfg(not(target_arch = "wasm32"))] app.add_system_to_stage( diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 7a3fcd3aa9508..ca30274822397 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -45,6 +45,8 @@ impl Plugin for PipelinedRenderingPlugin { if app.get_sub_app(RenderApp).is_err() { return; } + app.insert_resource(MainThreadExecutor::new()); + let mut sub_app = App::new(); sub_app.add_stage( RenderExtractStage::BeforeIoAfterRenderStart, @@ -63,7 +65,9 @@ impl Plugin for PipelinedRenderingPlugin { let (app_to_render_sender, app_to_render_receiver) = async_channel::bounded::(1); let (render_to_app_sender, render_to_app_receiver) = async_channel::bounded::(1); - let mut render_app = app.remove_sub_app(RenderApp).unwrap(); + let mut render_app = app + .remove_sub_app(RenderApp) + .expect("Unable to get RenderApp. Another plugin may have remove the RenderApp before PipelinedRenderingPlugin"); // clone main thread executor to render world let executor = app.world.get_resource::().unwrap(); From a3b1929fc47988ada428e0ccb7f5a360f486631a Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 14 Nov 2022 20:57:02 -0800 Subject: [PATCH 29/55] remove unnecessary sync bound --- crates/bevy_app/src/app.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index face53c99675e..b0cfba175969e 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -67,7 +67,7 @@ pub struct App { /// the application's event loop and advancing the [`Schedule`]. /// Typically, it is not configured manually, but set by one of Bevy's built-in plugins. /// See `bevy::winit::WinitPlugin` and [`ScheduleRunnerPlugin`](crate::schedule_runner::ScheduleRunnerPlugin). - pub runner: Box, // Send + Sync bound is only required to make App Send + Sync + pub runner: Box, // Send + Sync bound is only required to make App Send /// A container of [`Stage`]s set to be run in a linear order. pub schedule: Schedule, sub_apps: HashMap, @@ -89,7 +89,7 @@ impl Debug for App { pub struct SubApp { /// The [`SubApp`]'s instance of [`App`] pub app: App, - extract: Box, // Send + Sync bound is only required to make SubApp send sync + extract: Box, // Send bound is only required to make SubApp send } impl SubApp { @@ -823,7 +823,7 @@ impl App { /// App::new() /// .set_runner(my_runner); /// ``` - pub fn set_runner(&mut self, run_fn: impl Fn(App) + 'static + Send + Sync) -> &mut Self { + pub fn set_runner(&mut self, run_fn: impl Fn(App) + 'static + Send) -> &mut Self { self.runner = Box::new(run_fn); self } @@ -1015,7 +1015,7 @@ impl App { &mut self, label: impl AppLabel, app: App, - extract: impl Fn(&mut World, &mut App) + 'static + Send + Sync, + extract: impl Fn(&mut World, &mut App) + 'static + Send, ) -> &mut Self { self.sub_apps.insert( label.as_label(), From 32270e00c2a6901e090397b221b67b85e21f4bb1 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 15 Nov 2022 13:28:45 -0800 Subject: [PATCH 30/55] change scope to not tick global executor when running parallel executor --- crates/bevy_core/src/task_pool_options.rs | 4 +- .../src/schedule/executor_parallel.rs | 66 ++++++++++--------- crates/bevy_render/src/pipelined_rendering.rs | 34 +++++----- crates/bevy_tasks/src/lib.rs | 4 +- crates/bevy_tasks/src/task_pool.rs | 17 +++-- ..._thread_executor.rs => thread_executor.rs} | 4 +- examples/3d/3d_scene.rs | 10 ++- 7 files changed, 79 insertions(+), 60 deletions(-) rename crates/bevy_tasks/src/{main_thread_executor.rs => thread_executor.rs} (95%) diff --git a/crates/bevy_core/src/task_pool_options.rs b/crates/bevy_core/src/task_pool_options.rs index 05624c61fdc30..4537354a69c05 100644 --- a/crates/bevy_core/src/task_pool_options.rs +++ b/crates/bevy_core/src/task_pool_options.rs @@ -57,14 +57,14 @@ impl Default for TaskPoolOptions { min_total_threads: 1, max_total_threads: std::usize::MAX, - // Use 10% of cores for IO, at least 1, no more than 4 + // Use 25% of cores for IO, at least 1, no more than 4 io: TaskPoolThreadAssignmentPolicy { min_threads: 1, max_threads: 4, percent: 0.25, }, - // Use 10% of cores for async compute, at least 1, no more than 4 + // Use 25% of cores for async compute, at least 1, no more than 4 async_compute: TaskPoolThreadAssignmentPolicy { min_threads: 1, max_threads: 4, diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index eb51487680e82..0eff0ac89e0a8 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -148,40 +148,44 @@ impl ParallelSystemExecutor for ParallelExecutor { .get_resource::() .map(|e| e.0.clone()); - ComputeTaskPool::init(TaskPool::default).scope_with_executor(thread_executor, |scope| { - self.prepare_systems(scope, systems, world); - if self.should_run.count_ones(..) == 0 { - return; - } - let parallel_executor = async { - // All systems have been ran if there are no queued or running systems. - while 0 != self.queued.count_ones(..) + self.running.count_ones(..) { - self.process_queued_systems(); - // Avoid deadlocking if no systems were actually started. - if self.running.count_ones(..) != 0 { - // Wait until at least one system has finished. - let index = self - .finish_receiver - .recv() - .await - .unwrap_or_else(|error| unreachable!("{}", error)); - self.process_finished_system(index); - // Gather other systems than may have finished. - while let Ok(index) = self.finish_receiver.try_recv() { + ComputeTaskPool::init(TaskPool::default).scope_with_executor( + false, + thread_executor, + |scope| { + self.prepare_systems(scope, systems, world); + if self.should_run.count_ones(..) == 0 { + return; + } + let parallel_executor = async { + // All systems have been ran if there are no queued or running systems. + while 0 != self.queued.count_ones(..) + self.running.count_ones(..) { + self.process_queued_systems(); + // Avoid deadlocking if no systems were actually started. + if self.running.count_ones(..) != 0 { + // Wait until at least one system has finished. + let index = self + .finish_receiver + .recv() + .await + .unwrap_or_else(|error| unreachable!("{}", error)); self.process_finished_system(index); + // Gather other systems than may have finished. + while let Ok(index) = self.finish_receiver.try_recv() { + self.process_finished_system(index); + } + // At least one system has finished, so active access is outdated. + self.rebuild_active_access(); } - // At least one system has finished, so active access is outdated. - self.rebuild_active_access(); + self.update_counters_and_queue_systems(); } - self.update_counters_and_queue_systems(); - } - }; - #[cfg(feature = "trace")] - let span = bevy_utils::tracing::info_span!("parallel executor"); - #[cfg(feature = "trace")] - let parallel_executor = parallel_executor.instrument(span); - scope.spawn(parallel_executor); - }); + }; + #[cfg(feature = "trace")] + let span = bevy_utils::tracing::info_span!("parallel executor"); + #[cfg(feature = "trace")] + let parallel_executor = parallel_executor.instrument(span); + scope.spawn(parallel_executor); + }, + ); } } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index ca30274822397..1fb70fcb87808 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -7,8 +7,6 @@ use bevy_ecs::{ world::{Mut, World}, }; use bevy_tasks::ComputeTaskPool; -#[cfg(feature = "trace")] -use bevy_utils::tracing::Instrument; use crate::RenderApp; @@ -47,7 +45,7 @@ impl Plugin for PipelinedRenderingPlugin { } app.insert_resource(MainThreadExecutor::new()); - let mut sub_app = App::new(); + let mut sub_app = App::empty(); sub_app.add_stage( RenderExtractStage::BeforeIoAfterRenderStart, SystemStage::parallel(), @@ -78,20 +76,24 @@ impl Plugin for PipelinedRenderingPlugin { app.insert_resource(MainToRenderAppSender(app_to_render_sender)); app.insert_resource(RenderToMainAppReceiver(render_to_app_receiver)); - let render_task = async move { + std::thread::spawn(move || { + #[cfg(feature = "trace")] + let _span = bevy_utils::tracing::info_span!("render thread").entered(); + loop { - let recv_task = app_to_render_receiver.recv(); - let mut sub_app = recv_task.await.unwrap(); - sub_app.run(); - render_to_app_sender.send(sub_app).await.unwrap(); + // run a scope here to allow main world to use this thread while it's waiting for the render app + let mut render_app = ComputeTaskPool::get() + .scope(|s| { + s.spawn(async { app_to_render_receiver.recv().await.unwrap() }); + }) + .pop() + .unwrap(); + #[cfg(feature = "trace")] + let _span = bevy_utils::tracing::info_span!("render app").entered(); + render_app.run(); + render_to_app_sender.send_blocking(render_app).unwrap(); } - }; - #[cfg(feature = "trace")] - let span = bevy_utils::tracing::info_span!("render app"); - #[cfg(feature = "trace")] - let render_task = render_task.instrument(span); - - ComputeTaskPool::get().spawn(render_task).detach(); + }); } } @@ -102,7 +104,7 @@ fn update_rendering(app_world: &mut World, _sub_app: &mut App) { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. let mut render_app = ComputeTaskPool::get() - .scope_with_executor(Some(main_thread_executor.0.clone()), |s| { + .scope_with_executor(true, Some(main_thread_executor.0.clone()), |s| { s.spawn(async { let receiver = world.get_resource::().unwrap(); receiver.0.recv().await.unwrap() diff --git a/crates/bevy_tasks/src/lib.rs b/crates/bevy_tasks/src/lib.rs index e3e0250a2e568..f146ca2c7f84b 100644 --- a/crates/bevy_tasks/src/lib.rs +++ b/crates/bevy_tasks/src/lib.rs @@ -23,9 +23,9 @@ pub use usages::tick_global_task_pools_on_main_thread; pub use usages::{AsyncComputeTaskPool, ComputeTaskPool, IoTaskPool}; #[cfg(not(target_arch = "wasm32"))] -mod main_thread_executor; +mod thread_executor; #[cfg(not(target_arch = "wasm32"))] -pub use main_thread_executor::ThreadExecutor; +pub use thread_executor::ThreadExecutor; mod iter; pub use iter::ParallelIterator; diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 2dd36877a1776..636e2f9738bb6 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -11,7 +11,7 @@ use concurrent_queue::ConcurrentQueue; use futures_lite::{future, FutureExt}; use crate::Task; -use crate::{main_thread_executor::ThreadSpawner, ThreadExecutor}; +use crate::{thread_executor::ThreadSpawner, ThreadExecutor}; /// Used to create a [`TaskPool`] #[derive(Debug, Default, Clone)] @@ -238,7 +238,7 @@ impl TaskPool { F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { - self.scope_with_executor(None, f) + self.scope_with_executor(true, None, f) } /// This allows passing an external executor to spawn tasks on. When you pass an external executor @@ -246,6 +246,7 @@ impl TaskPool { /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, + tick_task_pool_executor: bool, thread_executor: Option>, f: F, ) -> Vec @@ -297,14 +298,20 @@ impl TaskPool { results }; - if let Some(main_thread_ticker) = thread_executor.ticker() { + if let Some(thread_ticker) = thread_executor.ticker() { let tick_forever = async move { loop { - main_thread_ticker.tick().await; + thread_ticker.tick().await; } }; - tick_forever.or(get_results).await + if tick_task_pool_executor { + executor.run(tick_forever).or(get_results).await + } else { + tick_forever.or(get_results).await + } + } else if tick_task_pool_executor { + executor.run(get_results).await } else { get_results.await } diff --git a/crates/bevy_tasks/src/main_thread_executor.rs b/crates/bevy_tasks/src/thread_executor.rs similarity index 95% rename from crates/bevy_tasks/src/main_thread_executor.rs rename to crates/bevy_tasks/src/thread_executor.rs index bb91faaf91626..9f587d33f85ee 100644 --- a/crates/bevy_tasks/src/main_thread_executor.rs +++ b/crates/bevy_tasks/src/thread_executor.rs @@ -29,13 +29,13 @@ impl ThreadExecutor { Self::default() } - /// Gets the `[MainThreadSpawner]` for the thread executor. + /// Gets the `[ThreadSpawner]` for the thread executor. /// Use this to spawn tasks that run on the thread this was instatiated on. pub fn spawner(&self) -> ThreadSpawner<'static> { ThreadSpawner(self.executor.clone()) } - /// Gets the `[MainThreadTicker]` for this executor. + /// Gets the `[ThreadTicker]` for this executor. /// Use this to tick the executor. /// It only returns the ticker if it's on the thread the executor was created on /// and returns `None` otherwise. diff --git a/examples/3d/3d_scene.rs b/examples/3d/3d_scene.rs index b2db6e3a43c30..0786747de503b 100644 --- a/examples/3d/3d_scene.rs +++ b/examples/3d/3d_scene.rs @@ -1,10 +1,16 @@ //! A simple 3D scene with light shining over a cube sitting on a plane. -use bevy::prelude::*; +use bevy::{prelude::*, window::PresentMode}; fn main() { App::new() - .add_plugins(DefaultPlugins) + .add_plugins(DefaultPlugins.set(WindowPlugin { + window: WindowDescriptor { + present_mode: PresentMode::AutoNoVsync, + ..default() + }, + ..default() + })) .add_startup_system(setup) .run(); } From 6e3837e190bb907f4a3d2556694eaeaf230e535a Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 15 Nov 2022 15:11:14 -0800 Subject: [PATCH 31/55] fix wasm --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index 61fd75a3472d2..ed2227540b7f8 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -77,7 +77,7 @@ impl TaskPool { F: for<'scope> FnOnce(&'env mut Scope<'scope, 'env, T>), T: Send + 'static, { - self.scope_with_executor(None, f) + self.scope_with_executor(false, None, f) } /// Allows spawning non-`static futures on the thread pool. The function takes a callback, @@ -87,6 +87,7 @@ impl TaskPool { /// This is similar to `rayon::scope` and `crossbeam::scope` pub fn scope_with_executor<'env, F, T>( &self, + _tick_task_pool_executor: bool, _thread_executor: Option>, f: F, ) -> Vec From 27d095c081a8985b27dad66f4f5f138149a0201c Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 15 Nov 2022 16:33:42 -0800 Subject: [PATCH 32/55] move extract commands to render world --- crates/bevy_render/src/lib.rs | 27 +++++++++++++------ crates/bevy_render/src/pipelined_rendering.rs | 3 ++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 586e30ef4fab8..23a1458e5220f 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -70,6 +70,9 @@ pub enum RenderStage { /// running the next frame while rendering the current frame. Extract, + /// A stage for applying the commands from the [`Extract`] stage + ExtractCommands, + /// Prepare render resources from the extracted data for the GPU. Prepare, @@ -192,11 +195,12 @@ impl Plugin for RenderPlugin { // after access to the main world is removed // See also https://github.com/bevyengine/bevy/issues/5082 extract_stage.set_apply_buffers(false); - fn clear_entities(world: &mut World) { - world.clear_entities(); - } + + let mut extract_commands_stage = SystemStage::parallel(); + extract_commands_stage.add_system(extract_commands.at_start()); render_app .add_stage(RenderStage::Extract, extract_stage) + .add_stage(RenderStage::ExtractCommands, extract_commands_stage) .add_stage(RenderStage::Prepare, SystemStage::parallel()) .add_stage(RenderStage::Queue, SystemStage::parallel()) .add_stage(RenderStage::PhaseSort, SystemStage::parallel()) @@ -312,10 +316,17 @@ fn extract(app_world: &mut World, render_app: &mut App) { let scratch_world = std::mem::replace(app_world, inserted_world.0); app_world.insert_resource(ScratchMainWorld(scratch_world)); - // Note: We apply buffers (read, Commands) after the `MainWorld` has been removed from the render app's world - // so that in future, pipelining will be able to do this too without any code relying on it. - // see - extract.0.apply_buffers(running_world); - render_app.world.insert_resource(extract); } + +// system for render app to apply the extract commands +fn extract_commands(world: &mut World) { + world.resource_scope(|world, mut extract_stage: Mut| { + extract_stage.0.apply_buffers(world); + }); +} + +// system for render app to clear entities +fn clear_entities(world: &mut World) { + world.clear_entities(); +} diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 1fb70fcb87808..2092ec0a21e03 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -22,7 +22,7 @@ pub enum RenderExtractStage { /// |-------------------------------------------------------------------| /// | | BeforeIoAfterRenderStart | winit events | main schedule | /// | extract |---------------------------------------------------------| - /// | | rendering schedule | + /// | | extract commands | rendering schedule | /// |-------------------------------------------------------------------| BeforeIoAfterRenderStart, } @@ -88,6 +88,7 @@ impl Plugin for PipelinedRenderingPlugin { }) .pop() .unwrap(); + #[cfg(feature = "trace")] let _span = bevy_utils::tracing::info_span!("render app").entered(); render_app.run(); From 41b0ccea7bfbac7acd13f599e279e4ba8bdd58d6 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 15 Nov 2022 16:55:06 -0800 Subject: [PATCH 33/55] clean up --- crates/bevy_app/src/app.rs | 4 ++-- crates/bevy_render/src/lib.rs | 2 +- crates/bevy_render/src/pipelined_rendering.rs | 2 +- examples/3d/3d_scene.rs | 10 ++-------- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index b0cfba175969e..26b090e29be46 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -67,7 +67,7 @@ pub struct App { /// the application's event loop and advancing the [`Schedule`]. /// Typically, it is not configured manually, but set by one of Bevy's built-in plugins. /// See `bevy::winit::WinitPlugin` and [`ScheduleRunnerPlugin`](crate::schedule_runner::ScheduleRunnerPlugin). - pub runner: Box, // Send + Sync bound is only required to make App Send + pub runner: Box, // Send bound is required to make App Send /// A container of [`Stage`]s set to be run in a linear order. pub schedule: Schedule, sub_apps: HashMap, @@ -93,7 +93,7 @@ pub struct SubApp { } impl SubApp { - /// runs the `SubApp` with its runner + /// run the `SubApp`'s schedule pub fn run(&mut self) { self.app.schedule.run(&mut self.app.world); } diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 23a1458e5220f..c4bc27f9bdb03 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -281,7 +281,7 @@ impl Plugin for RenderPlugin { fn setup(&self, app: &mut App) { if let Ok(render_app) = app.get_sub_app_mut(RenderApp) { - // move stage to resource so render_app.run() doesn't run it. + // move the extract stage to a resource so render_app.run() does not run it. let stage = render_app .schedule .remove_stage(RenderStage::Extract) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 2092ec0a21e03..56bf75e9b34f0 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -98,7 +98,7 @@ impl Plugin for PipelinedRenderingPlugin { } } -// This function is waits for the rendering world to be sent back, +// This function waits for the rendering world to be sent back, // runs extract, and then sends the rendering world back to the render thread. fn update_rendering(app_world: &mut World, _sub_app: &mut App) { app_world.resource_scope(|world, main_thread_executor: Mut| { diff --git a/examples/3d/3d_scene.rs b/examples/3d/3d_scene.rs index 0786747de503b..b2db6e3a43c30 100644 --- a/examples/3d/3d_scene.rs +++ b/examples/3d/3d_scene.rs @@ -1,16 +1,10 @@ //! A simple 3D scene with light shining over a cube sitting on a plane. -use bevy::{prelude::*, window::PresentMode}; +use bevy::prelude::*; fn main() { App::new() - .add_plugins(DefaultPlugins.set(WindowPlugin { - window: WindowDescriptor { - present_mode: PresentMode::AutoNoVsync, - ..default() - }, - ..default() - })) + .add_plugins(DefaultPlugins) .add_startup_system(setup) .run(); } From a4aa001545660937f65c235788fe4b8e39dda1bf Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 15 Nov 2022 18:34:00 -0800 Subject: [PATCH 34/55] use resource scope instead of removing resource --- crates/bevy_render/src/lib.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index c4bc27f9bdb03..e33f59507eb36 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -302,21 +302,20 @@ struct ScratchMainWorld(World); /// Executes the [`Extract`](RenderStage::Extract) stage of the renderer. /// This updates the render world with the extracted ECS data of the current frame. fn extract(app_world: &mut World, render_app: &mut App) { - let mut extract = render_app.world.remove_resource::().unwrap(); - - // temporarily add the app world to the render world as a resource - let scratch_world = app_world.remove_resource::().unwrap(); - let inserted_world = std::mem::replace(app_world, scratch_world.0); - let running_world = &mut render_app.world; - running_world.insert_resource(MainWorld(inserted_world)); - - extract.0.run(running_world); - // move the app world back, as if nothing happened. - let inserted_world = running_world.remove_resource::().unwrap(); - let scratch_world = std::mem::replace(app_world, inserted_world.0); - app_world.insert_resource(ScratchMainWorld(scratch_world)); - - render_app.world.insert_resource(extract); + render_app + .world + .resource_scope(|render_world, mut extract_stage: Mut| { + // temporarily add the app world to the render world as a resource + let scratch_world = app_world.remove_resource::().unwrap(); + let inserted_world = std::mem::replace(app_world, scratch_world.0); + render_world.insert_resource(MainWorld(inserted_world)); + + extract_stage.0.run(render_world); + // move the app world back, as if nothing happened. + let inserted_world = render_world.remove_resource::().unwrap(); + let scratch_world = std::mem::replace(app_world, inserted_world.0); + app_world.insert_resource(ScratchMainWorld(scratch_world)); + }); } // system for render app to apply the extract commands From 45d8f004f6ef35d55d3d29cfe4423aa96261bfe3 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 15 Nov 2022 22:18:32 -0800 Subject: [PATCH 35/55] remove incorrect comment --- crates/bevy_app/src/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 26b090e29be46..da36f8fcf9717 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -89,7 +89,7 @@ impl Debug for App { pub struct SubApp { /// The [`SubApp`]'s instance of [`App`] pub app: App, - extract: Box, // Send bound is only required to make SubApp send + extract: Box, } impl SubApp { From c57257b4225957b7f056b5c9b26e3f8adf04a751 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Wed, 16 Nov 2022 22:27:14 -0800 Subject: [PATCH 36/55] fix wasm builds --- crates/bevy_internal/src/default_plugins.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_internal/src/default_plugins.rs b/crates/bevy_internal/src/default_plugins.rs index f35d64d204477..808a43e8f9b15 100644 --- a/crates/bevy_internal/src/default_plugins.rs +++ b/crates/bevy_internal/src/default_plugins.rs @@ -65,7 +65,7 @@ impl PluginGroup for DefaultPlugins { // compressed texture formats .add(bevy_render::texture::ImagePlugin::default()); - #[cfg(not(target_arch = "wasm"))] + #[cfg(not(target_arch = "wasm32"))] { group = group .add(bevy_render::pipelined_rendering::PipelinedRenderingPlugin::default()); From 883fe29f0652a55cfc5f1f141c148b9872ab1398 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Sun, 4 Dec 2022 16:48:52 -0800 Subject: [PATCH 37/55] fix rebase issues --- crates/bevy_render/src/lib.rs | 2 -- crates/bevy_tasks/src/task_pool.rs | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index e33f59507eb36..3d48162c1e54d 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -38,12 +38,10 @@ pub mod prelude { use globals::GlobalsPlugin; pub use once_cell; -use prelude::ComputedVisibility; use crate::{ camera::CameraPlugin, mesh::MeshPlugin, - primitives::{CubemapFrusta, Frustum}, render_graph::RenderGraph, render_resource::{PipelineCache, Shader, ShaderLoader}, renderer::{render_system, RenderInstance}, diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 636e2f9738bb6..7bc0256e6a112 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -289,10 +289,10 @@ impl TaskPool { Vec::new() } else { future::block_on(async move { - let get_results = async { + let get_results = async { let mut results = Vec::with_capacity(scope.spawned.len()); while let Ok(task) = spawned_ref.pop() { - results.push(task.await); + results.push(task.await.unwrap()); } results From f506f74ca3c2961cbf64a41d02049a9aee59807d Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Sun, 4 Dec 2022 17:46:37 -0800 Subject: [PATCH 38/55] tick the task pool executor if there are no threads allocated --- crates/bevy_tasks/src/task_pool.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 7bc0256e6a112..cb55ba19d1ed5 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -298,6 +298,7 @@ impl TaskPool { results }; + let tick_task_pool_executor = tick_task_pool_executor || self.threads.is_empty(); if let Some(thread_ticker) = thread_executor.ticker() { let tick_forever = async move { loop { From 357549c25c9b6b26749d6c91c916408e6a2b9c8a Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Wed, 7 Dec 2022 13:11:19 -0800 Subject: [PATCH 39/55] call clear trackers on sub apps see #6878 --- crates/bevy_app/src/app.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index da36f8fcf9717..1a82e04ec4ed6 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -96,6 +96,7 @@ impl SubApp { /// run the `SubApp`'s schedule pub fn run(&mut self) { self.app.schedule.run(&mut self.app.world); + self.app.world.clear_trackers(); } /// extract data from main world to sub app From 128de37d77a441c1f2717b7c43c23564fb7583d0 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 12 Dec 2022 10:13:49 -0800 Subject: [PATCH 40/55] remove unnecessary system and rename another --- crates/bevy_render/src/lib.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index 3d48162c1e54d..34d101bf9e929 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -194,8 +194,10 @@ impl Plugin for RenderPlugin { // See also https://github.com/bevyengine/bevy/issues/5082 extract_stage.set_apply_buffers(false); + // This stage applies the commands from the extract stage while the render schedule + // is running in parallel with the main app. let mut extract_commands_stage = SystemStage::parallel(); - extract_commands_stage.add_system(extract_commands.at_start()); + extract_commands_stage.add_system(apply_extract_commands.at_start()); render_app .add_stage(RenderStage::Extract, extract_stage) .add_stage(RenderStage::ExtractCommands, extract_commands_stage) @@ -210,7 +212,7 @@ impl Plugin for RenderPlugin { ) .add_stage( RenderStage::Cleanup, - SystemStage::parallel().with_system(clear_entities.at_end()), + SystemStage::parallel().with_system(World::clear_entities.at_end()), ) .init_resource::() .insert_resource(RenderInstance(instance)) @@ -317,13 +319,8 @@ fn extract(app_world: &mut World, render_app: &mut App) { } // system for render app to apply the extract commands -fn extract_commands(world: &mut World) { +fn apply_extract_commands(world: &mut World) { world.resource_scope(|world, mut extract_stage: Mut| { extract_stage.0.apply_buffers(world); }); } - -// system for render app to clear entities -fn clear_entities(world: &mut World) { - world.clear_entities(); -} From cee9c53139b033fab99282e625aa55867862de4f Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 12 Dec 2022 10:46:25 -0800 Subject: [PATCH 41/55] remove unnecessary dependency on bevy_tasks --- crates/bevy_app/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/bevy_app/Cargo.toml b/crates/bevy_app/Cargo.toml index a4800e8881117..3705bde56eee5 100644 --- a/crates/bevy_app/Cargo.toml +++ b/crates/bevy_app/Cargo.toml @@ -20,7 +20,6 @@ bevy_derive = { path = "../bevy_derive", version = "0.9.0" } bevy_ecs = { path = "../bevy_ecs", version = "0.9.0", default-features = false } bevy_reflect = { path = "../bevy_reflect", version = "0.9.0", optional = true } bevy_utils = { path = "../bevy_utils", version = "0.9.0" } -bevy_tasks = { path = "../bevy_tasks", version = "0.9.0" } # other serde = { version = "1.0", features = ["derive"], optional = true } From e3267965e15f14be18eec942dcaf16807144eb05 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 13 Dec 2022 20:34:18 -0800 Subject: [PATCH 42/55] run executors forever even with panics --- crates/bevy_tasks/src/task_pool.rs | 54 +++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index cb55ba19d1ed5..c406659ab7c8b 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -2,6 +2,7 @@ use std::{ future::Future, marker::PhantomData, mem, + panic::AssertUnwindSafe, sync::Arc, thread::{self, JoinHandle}, }; @@ -300,19 +301,54 @@ impl TaskPool { let tick_task_pool_executor = tick_task_pool_executor || self.threads.is_empty(); if let Some(thread_ticker) = thread_executor.ticker() { - let tick_forever = async move { - loop { - thread_ticker.tick().await; - } - }; - if tick_task_pool_executor { - executor.run(tick_forever).or(get_results).await + let execute_forever = async move { + // we restart the executors if a task errors. if a scoped + // task errors it will panic the scope on the call to get_results + loop { + let tick_forever = async { + loop { + thread_ticker.tick().await; + } + }; + + // we don't care if it errors. If a scoped task errors it will propagate + // to get_results + let _result = AssertUnwindSafe(executor.run(tick_forever)) + .catch_unwind() + .await + .is_ok(); + } + }; + execute_forever.or(get_results).await } else { - tick_forever.or(get_results).await + let execute_forever = async { + loop { + let tick_forever = async { + loop { + thread_ticker.tick().await; + } + }; + + let _result = + AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); + } + }; + + execute_forever.or(get_results).await } } else if tick_task_pool_executor { - executor.run(get_results).await + let execute_forever = async { + loop { + let _result = + AssertUnwindSafe(executor.run(std::future::pending::<()>())) + .catch_unwind() + .await + .is_ok(); + } + }; + + execute_forever.or(get_results).await } else { get_results.await } From 0fe7234abe84d0c3699df36f4fe1855b69bdbeb5 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 10 Jan 2023 15:50:34 -0800 Subject: [PATCH 43/55] fix some merge issues --- .../src/schedule/executor_parallel.rs | 6 +-- crates/bevy_render/src/pipelined_rendering.rs | 2 +- crates/bevy_tasks/src/task_pool.rs | 47 ++++++++++++------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index 0eff0ac89e0a8..29a147749f57d 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -20,7 +20,7 @@ use scheduling_event::*; /// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread #[derive(Resource, Default)] -pub struct MainThreadExecutor(pub Arc); +pub struct MainThreadExecutor(pub Arc>); impl MainThreadExecutor { pub fn new() -> Self { @@ -144,9 +144,7 @@ impl ParallelSystemExecutor for ParallelExecutor { } } - let thread_executor = world - .get_resource::() - .map(|e| e.0.clone()); + let thread_executor = world.get_resource::().map(|e| &*e.0); ComputeTaskPool::init(TaskPool::default).scope_with_executor( false, diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 56bf75e9b34f0..069d8da8993fb 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -105,7 +105,7 @@ fn update_rendering(app_world: &mut World, _sub_app: &mut App) { // we use a scope here to run any main thread tasks that the render world still needs to run // while we wait for the render world to be received. let mut render_app = ComputeTaskPool::get() - .scope_with_executor(true, Some(main_thread_executor.0.clone()), |s| { + .scope_with_executor(true, Some(&*main_thread_executor.0), |s| { s.spawn(async { let receiver = world.get_resource::().unwrap(); receiver.0.recv().await.unwrap() diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 9882071a81bf3..62469eeead4e2 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -272,7 +272,8 @@ impl TaskPool { F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { - self.scope_with_executor(true, None, f) + Self::THREAD_EXECUTOR + .with(|thread_executor| self.scope_with_executor_inner(true, thread_executor, f)) } /// This allows passing an external executor to spawn tasks on. When you pass an external executor @@ -281,7 +282,26 @@ impl TaskPool { pub fn scope_with_executor<'env, F, T>( &self, tick_task_pool_executor: bool, - thread_executor: Option>, + thread_executor: Option<&ThreadExecutor>, + f: F, + ) -> Vec + where + F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), + T: Send + 'static, + { + if let Some(thread_executor) = thread_executor { + self.scope_with_executor_inner(tick_task_pool_executor, thread_executor, f) + } else { + Self::THREAD_EXECUTOR.with(|thread_executor| { + self.scope_with_executor_inner(tick_task_pool_executor, thread_executor, f) + }) + } + } + + fn scope_with_executor_inner<'env, F, T>( + &self, + tick_task_pool_executor: bool, + thread_executor: &ThreadExecutor, f: F, ) -> Vec where @@ -295,33 +315,28 @@ impl TaskPool { // transmute the lifetimes to 'env here to appease the compiler as it is unable to validate safety. let executor: &async_executor::Executor = &self.executor; let executor: &'env async_executor::Executor = unsafe { mem::transmute(executor) }; - - let thread_executor = if let Some(thread_executor) = thread_executor { - thread_executor - } else { - Arc::new(ThreadExecutor::new()) - }; - let thread_spawner = thread_executor.spawner(); - let thread_spawner: ThreadSpawner<'env> = unsafe { mem::transmute(thread_spawner) }; + let thread_executor: &'env ThreadExecutor<'env> = + unsafe { mem::transmute(thread_executor) }; let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); let spawned_ref: &'env ConcurrentQueue> = unsafe { mem::transmute(&spawned) }; let scope = Scope { executor, - thread_spawner, + thread_executor, spawned: spawned_ref, scope: PhantomData, env: PhantomData, }; - let scope_ref: &'env Scope<'_, 'env, T> = unsafe { mem::transmute(&scope) }; + let scope_ref: &'env Scope<'_, 'env, T> = unsafe { mem::transmute(&scope) }; - f(scope_ref); + f(scope_ref); - if spawned.is_empty() { - Vec::new() - } else { + if spawned.is_empty() { + Vec::new() + } else { + future::block_on(async move { let get_results = async { let mut results = Vec::with_capacity(spawned_ref.len()); while let Ok(task) = spawned_ref.pop() { From f3d5a0c15317012a90aa33e0bf6b951812eced39 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 10 Jan 2023 17:01:37 -0800 Subject: [PATCH 44/55] fix wasm build --- crates/bevy_tasks/src/single_threaded_task_pool.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index ed2227540b7f8..c9546c958b752 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -11,11 +11,11 @@ pub struct TaskPoolBuilder {} /// dummy struct for wasm #[derive(Default)] -pub struct ThreadExecutor; -impl ThreadExecutor { +pub struct ThreadExecutor<'a>(PhantomData<&'a ()>); +impl<'a> ThreadExecutor<'a> { /// creates a new `ThreadExecutor` pub fn new() -> Self { - Self + Self(PhantomData::default()) } } @@ -88,7 +88,7 @@ impl TaskPool { pub fn scope_with_executor<'env, F, T>( &self, _tick_task_pool_executor: bool, - _thread_executor: Option>, + _thread_executor: Option<&ThreadExecutor>, f: F, ) -> Vec where From 8cf5e652e2c250b77fa88931b1507df88d59154a Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 10 Jan 2023 20:20:59 -0800 Subject: [PATCH 45/55] Apply suggestions from code review Co-authored-by: Alice Cecile --- crates/bevy_app/src/app.rs | 2 +- crates/bevy_render/src/lib.rs | 2 +- crates/bevy_tasks/src/single_threaded_task_pool.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index df73d03fbfcde..60b73b232190e 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -1079,7 +1079,7 @@ impl App { self.sub_apps.insert(label.as_label(), sub_app); } - /// Removes a sub app from the app. Returns None if the label doesn't exist. + /// Removes a sub app from the app. Returns [`None`] if the label doesn't exist. pub fn remove_sub_app(&mut self, label: impl AppLabel) -> Option { self.sub_apps.remove(&label.as_label()) } diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index b1ae36637714f..8e1c8fd5cb0e0 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -231,7 +231,7 @@ impl Plugin for RenderPlugin { app.add_sub_app(RenderApp, render_app, move |app_world, render_app| { #[cfg(feature = "trace")] - let _render_span = bevy_utils::tracing::info_span!("extract main to render app").entered(); + let _render_span = bevy_utils::tracing::info_span!("extract main app to render subapp").entered(); { #[cfg(feature = "trace")] let _stage_span = diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index c9546c958b752..d33f65a38b2f9 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -13,7 +13,7 @@ pub struct TaskPoolBuilder {} #[derive(Default)] pub struct ThreadExecutor<'a>(PhantomData<&'a ()>); impl<'a> ThreadExecutor<'a> { - /// creates a new `ThreadExecutor` + /// Creates a new `ThreadExecutor` pub fn new() -> Self { Self(PhantomData::default()) } From 47c53649b54a90b135c0a5f8e8cfdd579324eab3 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Tue, 10 Jan 2023 21:02:13 -0800 Subject: [PATCH 46/55] resolve some review comments --- crates/bevy_app/src/app.rs | 9 +++- .../src/schedule/executor_parallel.rs | 8 +--- crates/bevy_render/src/pipelined_rendering.rs | 47 +++++++++++++++---- .../src/single_threaded_task_pool.rs | 6 ++- crates/bevy_tasks/src/task_pool.rs | 3 ++ 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 60b73b232190e..09769605b78a0 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -87,11 +87,16 @@ impl Debug for App { } } -/// Each `SubApp` has its own [`Schedule`] and [`World`], enabling a separation of concerns. +/// A [`SubApp`] contains its own [`Schedule`] and [`World`] separate from the main [`App`]. +/// This is useful for situations where data and data processing should be kept completely separate +/// from the main application. The primary use of this feature in bevy is to enable pipelined rendering. pub struct SubApp { /// The [`SubApp`]'s instance of [`App`] pub app: App, - extract: Box, + + /// A function that allows access to both the [`SubApp`] [`World`] and the main [`App`]. This is + /// useful for moving data between the sub app and the main app. + pub extract: Box, } impl SubApp { diff --git a/crates/bevy_ecs/src/schedule/executor_parallel.rs b/crates/bevy_ecs/src/schedule/executor_parallel.rs index 29a147749f57d..0db9627633ba1 100644 --- a/crates/bevy_ecs/src/schedule/executor_parallel.rs +++ b/crates/bevy_ecs/src/schedule/executor_parallel.rs @@ -19,7 +19,7 @@ use fixedbitset::FixedBitSet; use scheduling_event::*; /// New-typed [`ThreadExecutor`] [`Resource`] that is used to run systems on the main thread -#[derive(Resource, Default)] +#[derive(Resource, Default, Clone)] pub struct MainThreadExecutor(pub Arc>); impl MainThreadExecutor { @@ -28,12 +28,6 @@ impl MainThreadExecutor { } } -impl Clone for MainThreadExecutor { - fn clone(&self) -> Self { - MainThreadExecutor(self.0.clone()) - } -} - struct SystemSchedulingMetadata { /// Used to signal the system's task to start the system. start: Event, diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 069d8da8993fb..6224940a61333 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -14,16 +14,12 @@ use crate::RenderApp; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] pub struct RenderExtractApp; -/// Labels for stages in the sub app that syncs with the rendering task. +/// Labels for stages in the [`RenderExtractApp`] sub app. These will run after rendering has started. #[derive(Debug, Hash, PartialEq, Eq, Clone, StageLabel)] pub enum RenderExtractStage { - /// This stage runs after the render schedule starts, but before I/O processing and the main app schedule. - /// This can be useful for something like frame pacing. - /// |-------------------------------------------------------------------| - /// | | BeforeIoAfterRenderStart | winit events | main schedule | - /// | extract |---------------------------------------------------------| - /// | | extract commands | rendering schedule | - /// |-------------------------------------------------------------------| + /// When pipelined rendering is enabled this stage runs after the render schedule starts, but + /// before I/O processing and the main app schedule. This can be useful for something like + /// frame pacing. BeforeIoAfterRenderStart, } @@ -35,6 +31,41 @@ pub struct MainToRenderAppSender(pub Sender); #[derive(Resource)] pub struct RenderToMainAppReceiver(pub Receiver); +/// The [`PipelinedRenderingPlugin`] can be added to your application to enable pipelined rendering. +/// This moves rendering into a different thread, so that the Nth frame's rendering can +/// be run at the same time as the N + 1 frame's simulation. +/// +/// ```text +/// |--------------------|--------------------|--------------------|--------------------| +/// | simulation thread | frame 1 simulation | frame 2 simulation | frame 3 simulation | +/// |--------------------|--------------------|--------------------|--------------------| +/// | rendering thread | | frame 1 rendering | frame 2 rendering | +/// |--------------------|--------------------|--------------------|--------------------| +/// ``` +/// +/// The plugin is dependent on the [`crate::RenderApp`] added by [`crate::RenderPlugin`] and so must +/// be added after that plugin. If it is not added after, the plugin will do nothing. +/// +/// A single frame of execution looks something like below +/// +/// ```text +/// |-------------------------------------------------------------------| +/// | | BeforeIoAfterRenderStart | winit events | main schedule | +/// | extract |---------------------------------------------------------| +/// | | extract commands | rendering schedule | +/// |-------------------------------------------------------------------| +/// ``` +/// +/// - `extract` is the stage where data is copied from the main world to the render world. +/// This is run on the main app's thread. +/// - On the render thread, we first apply the `extract commands`. This is not run during extract, so the +/// main schedule can start sooner. +/// - Then the `rendering schedule` is run. See [crate::RenderStage] for the available stages. +/// - In parallel to the rendering thread we first run the [`RenderExtractStage::BeforeIoAfterRenderStart`] stage. By +/// default this stage is empty. But is useful if you need something to run before I/O processing. +/// - Next all the `winit events` are processed. +/// - And finally the `main app schedule` is run. +/// - Once both the `main app schedule` and the `render schedule` are finished running, `extract` is run again. #[derive(Default)] pub struct PipelinedRenderingPlugin; impl Plugin for PipelinedRenderingPlugin { diff --git a/crates/bevy_tasks/src/single_threaded_task_pool.rs b/crates/bevy_tasks/src/single_threaded_task_pool.rs index d33f65a38b2f9..9b77d8fd3bb2c 100644 --- a/crates/bevy_tasks/src/single_threaded_task_pool.rs +++ b/crates/bevy_tasks/src/single_threaded_task_pool.rs @@ -9,7 +9,11 @@ use std::{ #[derive(Debug, Default, Clone)] pub struct TaskPoolBuilder {} -/// dummy struct for wasm +/// This is a dummy struct for wasm support to provide the same api as with the multithreaded +/// task pool. In the case of the multithreaded task pool this struct is used to spawn +/// tasks on a specific thread. But the wasm task pool just calls +/// [`wasm_bindgen_futures::spawn_local`] for spawning which just runs tasks on the main thread +/// and so the [`ThreadExecutor`] does nothing. #[derive(Default)] pub struct ThreadExecutor<'a>(PhantomData<&'a ()>); impl<'a> ThreadExecutor<'a> { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 62469eeead4e2..c7fd8ee92af65 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -278,6 +278,7 @@ impl TaskPool { /// This allows passing an external executor to spawn tasks on. When you pass an external executor /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. + /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, @@ -289,6 +290,8 @@ impl TaskPool { F: for<'scope> FnOnce(&'scope Scope<'scope, 'env, T>), T: Send + 'static, { + // If a `thread_executor` is passed use that. Otherwise get the `thread_executor` stored + // in the `THREAD_EXECUTOR` thread local. if let Some(thread_executor) = thread_executor { self.scope_with_executor_inner(tick_task_pool_executor, thread_executor, f) } else { From e4af50ad0c5716186f86e3d797667a9a05662876 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Wed, 11 Jan 2023 13:51:39 -0800 Subject: [PATCH 47/55] add example to subapp docs --- crates/bevy_app/src/app.rs | 41 ++++++++++++++++++- crates/bevy_render/src/pipelined_rendering.rs | 14 +++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index 09769605b78a0..a9083222f5d72 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -90,11 +90,50 @@ impl Debug for App { /// A [`SubApp`] contains its own [`Schedule`] and [`World`] separate from the main [`App`]. /// This is useful for situations where data and data processing should be kept completely separate /// from the main application. The primary use of this feature in bevy is to enable pipelined rendering. +/// +/// # Example +/// +/// ```rust +/// # use bevy_app::{App, AppLabel}; +/// # use bevy_ecs::prelude::*; +/// +/// #[derive(Resource, Default)] +/// struct Val(pub i32); +/// +/// #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, AppLabel)] +/// struct ExampleApp; +/// +/// #[derive(Debug, Hash, PartialEq, Eq, Clone, StageLabel)] +/// struct ExampleStage; +/// +/// let mut app = App::empty(); +/// // initialize the main app with a value of 0; +/// app.insert_resource(Val(10)); +/// +/// // create a app with a resource and a single stage +/// let mut sub_app = App::empty(); +/// sub_app.insert_resource(Val(100)); +/// let mut example_stage = SystemStage::single_threaded(); +/// example_stage.add_system(|counter: Res| { +/// // since we assigned the value from the main world in extract +/// // we see that value instead of 100 +/// assert_eq!(counter.0, 10); +/// }); +/// sub_app.add_stage(ExampleStage, example_stage); +/// +/// // add the sub_app to the app +/// app.add_sub_app(ExampleApp, sub_app, |main_world, sub_app| { +/// sub_app.world.resource_mut::().0 = main_world.resource::().0; +/// }); +/// +/// // This will run the schedules once, since we're using the default runner +/// app.run(); +/// ``` pub struct SubApp { /// The [`SubApp`]'s instance of [`App`] pub app: App, - /// A function that allows access to both the [`SubApp`] [`World`] and the main [`App`]. This is + /// A function that allows access to both the [`SubApp`] [`World`] and the main [`App`]. This is /// useful for moving data between the sub app and the main app. pub extract: Box, } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 6224940a61333..6869aac1d5481 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -17,8 +17,8 @@ pub struct RenderExtractApp; /// Labels for stages in the [`RenderExtractApp`] sub app. These will run after rendering has started. #[derive(Debug, Hash, PartialEq, Eq, Clone, StageLabel)] pub enum RenderExtractStage { - /// When pipelined rendering is enabled this stage runs after the render schedule starts, but - /// before I/O processing and the main app schedule. This can be useful for something like + /// When pipelined rendering is enabled this stage runs after the render schedule starts, but + /// before I/O processing and the main app schedule. This can be useful for something like /// frame pacing. BeforeIoAfterRenderStart, } @@ -34,7 +34,7 @@ pub struct RenderToMainAppReceiver(pub Receiver); /// The [`PipelinedRenderingPlugin`] can be added to your application to enable pipelined rendering. /// This moves rendering into a different thread, so that the Nth frame's rendering can /// be run at the same time as the N + 1 frame's simulation. -/// +/// /// ```text /// |--------------------|--------------------|--------------------|--------------------| /// | simulation thread | frame 1 simulation | frame 2 simulation | frame 3 simulation | @@ -42,12 +42,12 @@ pub struct RenderToMainAppReceiver(pub Receiver); /// | rendering thread | | frame 1 rendering | frame 2 rendering | /// |--------------------|--------------------|--------------------|--------------------| /// ``` -/// +/// /// The plugin is dependent on the [`crate::RenderApp`] added by [`crate::RenderPlugin`] and so must /// be added after that plugin. If it is not added after, the plugin will do nothing. -/// +/// /// A single frame of execution looks something like below -/// +/// /// ```text /// |-------------------------------------------------------------------| /// | | BeforeIoAfterRenderStart | winit events | main schedule | @@ -55,7 +55,7 @@ pub struct RenderToMainAppReceiver(pub Receiver); /// | | extract commands | rendering schedule | /// |-------------------------------------------------------------------| /// ``` -/// +/// /// - `extract` is the stage where data is copied from the main world to the render world. /// This is run on the main app's thread. /// - On the render thread, we first apply the `extract commands`. This is not run during extract, so the From a78ae5207dd19e5ba2e1a56cc7193556a9161a5d Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Wed, 11 Jan 2023 14:06:12 -0800 Subject: [PATCH 48/55] fix ci --- crates/bevy_render/src/pipelined_rendering.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 6869aac1d5481..40224cb10b2b1 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -60,7 +60,7 @@ pub struct RenderToMainAppReceiver(pub Receiver); /// This is run on the main app's thread. /// - On the render thread, we first apply the `extract commands`. This is not run during extract, so the /// main schedule can start sooner. -/// - Then the `rendering schedule` is run. See [crate::RenderStage] for the available stages. +/// - Then the `rendering schedule` is run. See [`crate::RenderStage`] for the available stages. /// - In parallel to the rendering thread we first run the [`RenderExtractStage::BeforeIoAfterRenderStart`] stage. By /// default this stage is empty. But is useful if you need something to run before I/O processing. /// - Next all the `winit events` are processed. From 39b636ca3e5dc7646c5d33c85f9633d2afb9502d Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Wed, 11 Jan 2023 14:23:43 -0800 Subject: [PATCH 49/55] refactor scope executors into separate functions --- crates/bevy_tasks/src/task_pool.rs | 113 +++++++++++++++++------------ 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index c7fd8ee92af65..9db3670da50ab 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -11,7 +11,10 @@ use async_task::FallibleTask; use concurrent_queue::ConcurrentQueue; use futures_lite::{future, FutureExt}; -use crate::{thread_executor::ThreadExecutor, Task}; +use crate::{ + thread_executor::{ThreadExecutor, ThreadExecutorTicker}, + Task, +}; struct CallOnDrop(Option>); @@ -345,60 +348,18 @@ impl TaskPool { while let Ok(task) = spawned_ref.pop() { results.push(task.await.unwrap()); } - results }; let tick_task_pool_executor = tick_task_pool_executor || self.threads.is_empty(); if let Some(thread_ticker) = thread_executor.ticker() { if tick_task_pool_executor { - let execute_forever = async move { - // we restart the executors if a task errors. if a scoped - // task errors it will panic the scope on the call to get_results - loop { - let tick_forever = async { - loop { - thread_ticker.tick().await; - } - }; - - // we don't care if it errors. If a scoped task errors it will propagate - // to get_results - let _result = AssertUnwindSafe(executor.run(tick_forever)) - .catch_unwind() - .await - .is_ok(); - } - }; - execute_forever.or(get_results).await + Self::execute_local_global(thread_ticker, executor, get_results).await } else { - let execute_forever = async { - loop { - let tick_forever = async { - loop { - thread_ticker.tick().await; - } - }; - - let _result = - AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); - } - }; - - execute_forever.or(get_results).await + Self::execute_local(thread_ticker, get_results).await } } else if tick_task_pool_executor { - let execute_forever = async { - loop { - let _result = - AssertUnwindSafe(executor.run(std::future::pending::<()>())) - .catch_unwind() - .await - .is_ok(); - } - }; - - execute_forever.or(get_results).await + Self::execute_global(executor, get_results).await } else { get_results.await } @@ -406,6 +367,66 @@ impl TaskPool { } } + #[inline] + async fn execute_local_global<'scope, 'ticker, T>( + thread_ticker: ThreadExecutorTicker<'scope, 'ticker>, + executor: &'scope async_executor::Executor<'scope>, + get_results: impl Future>, + ) -> Vec { + // we restart the executors if a task errors. if a scoped + // task errors it will panic the scope on the call to get_results + let execute_forever = async move { + loop { + let tick_forever = async { + loop { + thread_ticker.tick().await; + } + }; + // we don't care if it errors. If a scoped task errors it will propagate + // to get_results + let _result = AssertUnwindSafe(executor.run(tick_forever)) + .catch_unwind() + .await + .is_ok(); + } + }; + execute_forever.or(get_results).await + } + + #[inline] + async fn execute_local<'scope, 'ticker, T>( + thread_ticker: ThreadExecutorTicker<'scope, 'ticker>, + get_results: impl Future>, + ) -> Vec { + let execute_forever = async { + loop { + let tick_forever = async { + loop { + thread_ticker.tick().await; + } + }; + let _result = AssertUnwindSafe(tick_forever).catch_unwind().await.is_ok(); + } + }; + execute_forever.or(get_results).await + } + + #[inline] + async fn execute_global<'scope, T>( + executor: &'scope async_executor::Executor<'scope>, + get_results: impl Future>, + ) -> Vec { + let execute_forever = async { + loop { + let _result = AssertUnwindSafe(executor.run(std::future::pending::<()>())) + .catch_unwind() + .await + .is_ok(); + } + }; + execute_forever.or(get_results).await + } + /// Spawns a static future onto the thread pool. The returned Task is a future. It can also be /// cancelled and "detached" allowing it to continue running without having to be polled by the /// end-user. From 2f1c317ffbbc7b2d2766effe4050baf6d835a113 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Wed, 11 Jan 2023 19:25:50 -0800 Subject: [PATCH 50/55] add tracing spans for all sub apps --- crates/bevy_app/src/app.rs | 4 +++- crates/bevy_render/src/pipelined_rendering.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/bevy_app/src/app.rs b/crates/bevy_app/src/app.rs index a9083222f5d72..eba56f52f617b 100644 --- a/crates/bevy_app/src/app.rs +++ b/crates/bevy_app/src/app.rs @@ -211,7 +211,9 @@ impl App { let _bevy_frame_update_span = info_span!("main app").entered(); self.schedule.run(&mut self.world); } - for sub_app in self.sub_apps.values_mut() { + for (_label, sub_app) in self.sub_apps.iter_mut() { + #[cfg(feature = "trace")] + let _sub_app_span = info_span!("sub app", name = ?_label).entered(); sub_app.extract(&mut self.world); sub_app.run(); } diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 40224cb10b2b1..9001f2c9bbbac 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -121,7 +121,8 @@ impl Plugin for PipelinedRenderingPlugin { .unwrap(); #[cfg(feature = "trace")] - let _span = bevy_utils::tracing::info_span!("render app").entered(); + let _sub_app_span = + bevy_utils::tracing::info_span!("sub app", name = ?RenderApp).entered(); render_app.run(); render_to_app_sender.send_blocking(render_app).unwrap(); } From 92d123c7a4e9a313cf5cc40cf0a365539293e29b Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 16 Jan 2023 11:12:06 -0800 Subject: [PATCH 51/55] change world.get_resource to resource Co-authored-by: James Liu --- crates/bevy_render/src/pipelined_rendering.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index 9001f2c9bbbac..e253664e87ca2 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -148,7 +148,7 @@ fn update_rendering(app_world: &mut World, _sub_app: &mut App) { render_app.extract(world); - let sender = world.get_resource::().unwrap(); + let sender = world.resource::(); sender.0.send_blocking(render_app).unwrap(); }); } From 1d6a5b9dc59e0088d38b23e477332ddfc511afe5 Mon Sep 17 00:00:00 2001 From: Michael Hsu Date: Mon, 16 Jan 2023 11:45:27 -0800 Subject: [PATCH 52/55] add doc for tick_task_pool_executor --- crates/bevy_tasks/src/task_pool.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 9db3670da50ab..4713ccc721a5b 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -282,6 +282,11 @@ impl TaskPool { /// This allows passing an external executor to spawn tasks on. When you pass an external executor /// [`Scope::spawn_on_scope`] spawns is then run on the thread that [`ThreadExecutor`] is being ticked on. /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. + /// + /// When `tick_task_pool_executor` is set to `true`, the multithreaded task stealing executor is ticked on the scope + /// thread. Disabling this can be useful when finishing the scope is latancy sensitive. Pulling tasks from + /// global excutor can run tasks unrelated to the scoape and delay when the scope returns. + /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( &self, From 2faeb9befee2d75e4199d82a0a0b719ccf319866 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 16 Jan 2023 12:56:05 -0800 Subject: [PATCH 53/55] fix spelling Co-authored-by: James Liu --- crates/bevy_tasks/src/task_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 4713ccc721a5b..8533341500b60 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -284,7 +284,7 @@ impl TaskPool { /// If [`None`] is passed the scope will use a [`ThreadExecutor`] that is ticked on the current thread. /// /// When `tick_task_pool_executor` is set to `true`, the multithreaded task stealing executor is ticked on the scope - /// thread. Disabling this can be useful when finishing the scope is latancy sensitive. Pulling tasks from + /// thread. Disabling this can be useful when finishing the scope is latency sensitive. Pulling tasks from /// global excutor can run tasks unrelated to the scoape and delay when the scope returns. /// /// See [`Self::scope`] for more details in general about how scopes work. From 04440d5df0472d0abcb96f8f36e007ee74d19c19 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 18 Jan 2023 12:29:15 -0800 Subject: [PATCH 54/55] Apply suggestions from code review Co-authored-by: Robert Swain --- crates/bevy_render/src/pipelined_rendering.rs | 5 +++-- crates/bevy_tasks/src/task_pool.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/bevy_render/src/pipelined_rendering.rs b/crates/bevy_render/src/pipelined_rendering.rs index e253664e87ca2..63c0a0cfb74b6 100644 --- a/crates/bevy_render/src/pipelined_rendering.rs +++ b/crates/bevy_render/src/pipelined_rendering.rs @@ -68,6 +68,7 @@ pub struct RenderToMainAppReceiver(pub Receiver); /// - Once both the `main app schedule` and the `render schedule` are finished running, `extract` is run again. #[derive(Default)] pub struct PipelinedRenderingPlugin; + impl Plugin for PipelinedRenderingPlugin { fn build(&self, app: &mut App) { // Don't add RenderExtractApp if RenderApp isn't initialized. @@ -96,7 +97,7 @@ impl Plugin for PipelinedRenderingPlugin { let mut render_app = app .remove_sub_app(RenderApp) - .expect("Unable to get RenderApp. Another plugin may have remove the RenderApp before PipelinedRenderingPlugin"); + .expect("Unable to get RenderApp. Another plugin may have removed the RenderApp before PipelinedRenderingPlugin"); // clone main thread executor to render world let executor = app.world.get_resource::().unwrap(); @@ -130,7 +131,7 @@ impl Plugin for PipelinedRenderingPlugin { } } -// This function waits for the rendering world to be sent back, +// This function waits for the rendering world to be received, // runs extract, and then sends the rendering world back to the render thread. fn update_rendering(app_world: &mut World, _sub_app: &mut App) { app_world.resource_scope(|world, main_thread_executor: Mut| { diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index 8533341500b60..d29e00134d3ca 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -285,7 +285,7 @@ impl TaskPool { /// /// When `tick_task_pool_executor` is set to `true`, the multithreaded task stealing executor is ticked on the scope /// thread. Disabling this can be useful when finishing the scope is latency sensitive. Pulling tasks from - /// global excutor can run tasks unrelated to the scoape and delay when the scope returns. + /// global excutor can run tasks unrelated to the scope and delay when the scope returns. /// /// See [`Self::scope`] for more details in general about how scopes work. pub fn scope_with_executor<'env, F, T>( From 0ebbde7e000067e67b3abfd53a40b74c7c5ec0a3 Mon Sep 17 00:00:00 2001 From: Mike Hsu Date: Thu, 19 Jan 2023 15:20:18 -0800 Subject: [PATCH 55/55] fix transmute --- crates/bevy_tasks/src/task_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_tasks/src/task_pool.rs b/crates/bevy_tasks/src/task_pool.rs index d29e00134d3ca..250bfba91f72c 100644 --- a/crates/bevy_tasks/src/task_pool.rs +++ b/crates/bevy_tasks/src/task_pool.rs @@ -328,7 +328,7 @@ impl TaskPool { let executor: &'env async_executor::Executor = unsafe { mem::transmute(executor) }; let thread_executor: &'env ThreadExecutor<'env> = unsafe { mem::transmute(thread_executor) }; - let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); + let spawned: ConcurrentQueue> = ConcurrentQueue::unbounded(); let spawned_ref: &'env ConcurrentQueue> = unsafe { mem::transmute(&spawned) };