UI Text picking

Cargo.toml

@@ -4001,6 +4001,19 @@ description = "Demonstrates picking sprites and sprite atlases"
 category = "Picking"
 wasm = true
 
+[[example]]
+name = "text_picking"
+path = "examples/picking/text_picking.rs"
+doc-scrape-examples = true
+required-features = ["bevy_ui_picking_backend"]
+
+[package.metadata.example.text_picking]
+name = "Text Picking"
+description = "Demonstrates picking text"
+category = "Picking"
+wasm = true
+required-features = ["bevy_ui_picking_backend"]
+
 [[example]]
 name = "debug_picking"
 path = "examples/picking/debug_picking.rs"

crates/bevy_text/Cargo.toml

@@ -10,6 +10,7 @@ keywords = ["bevy"]
 
 [features]
 default_font = []
+picking = ["bevy_picking"]
 
 [dependencies]
 # bevy
@@ -21,6 +22,7 @@ bevy_ecs = { path = "../bevy_ecs", version = "0.16.0-dev" }
 bevy_image = { path = "../bevy_image", version = "0.16.0-dev" }
 bevy_log = { path = "../bevy_log", version = "0.16.0-dev" }
 bevy_math = { path = "../bevy_math", version = "0.16.0-dev" }
+bevy_picking = { path = "../bevy_picking", version = "0.16.0-dev", optional = true }
 bevy_reflect = { path = "../bevy_reflect", version = "0.16.0-dev", features = [
   "bevy",
 ] }
@@ -42,6 +44,7 @@ smallvec = "1.13"
 unicode-bidi = "0.3.13"
 sys-locale = "0.3.0"
 tracing = { version = "0.1", default-features = false, features = ["std"] }
+radsort = "0.1"
 
 [dev-dependencies]
 approx = "0.5.1"

crates/bevy_text/src/glyph.rs

@@ -20,24 +20,12 @@ pub struct PositionedGlyph {
     pub atlas_info: GlyphAtlasInfo,
     /// The index of the glyph in the [`ComputedTextBlock`](crate::ComputedTextBlock)'s tracked spans.
     pub span_index: usize,
-    /// TODO: In order to do text editing, we need access to the size of glyphs and their index in the associated String.
-    /// For example, to figure out where to place the cursor in an input box from the mouse's position.
-    /// Without this, it's only possible in texts where each glyph is one byte. Cosmic text has methods for this
-    /// cosmic-texts [hit detection](https://pop-os.github.io/cosmic-text/cosmic_text/struct.Buffer.html#method.hit)
-    byte_index: usize,
-}
-
-impl PositionedGlyph {
-    /// Creates a new [`PositionedGlyph`]
-    pub fn new(position: Vec2, size: Vec2, atlas_info: GlyphAtlasInfo, span_index: usize) -> Self {
-        Self {
-            position,
-            size,
-            atlas_info,
-            span_index,
-            byte_index: 0,
-        }
-    }
+    /// The index of the glyph's line.
+    pub line_index: usize,
+    /// The byte index of the glyph in it's line.
+    pub byte_index: usize,
+    /// The byte length of the glyph.
+    pub byte_length: usize,
 }
 
 /// Information about a glyph in an atlas.

crates/bevy_text/src/lib.rs

@@ -38,10 +38,16 @@ mod font_atlas;
 mod font_atlas_set;
 mod font_loader;
 mod glyph;
+#[cfg(feature = "picking")]
+mod picking_backend;
 mod pipeline;
 mod text;
 mod text2d;
 mod text_access;
+#[cfg(feature = "picking")]
+pub mod text_picking_backend;
+#[cfg(feature = "picking")]
+pub mod text_pointer;
 
 pub use bounds::*;
 pub use error::*;
@@ -154,5 +160,14 @@ impl Plugin for TextPlugin {
             "FiraMono-subset.ttf",
             |bytes: &[u8], _path: String| { Font::try_from_bytes(bytes.to_vec()).unwrap() }
         );
+
+        #[cfg(feature = "picking")]
+        {
+            app.add_plugins((
+                text_pointer::plugin,
+                text_picking_backend::plugin,
+                picking_backend::Text2dPickingPlugin,
+            ));
+        }
     }
 }

crates/bevy_text/src/picking_backend.rs

@@ -0,0 +1,220 @@
+use bevy_app::{App, Plugin, PreUpdate};
+use bevy_ecs::component::Component;
+use bevy_ecs::prelude::ReflectResource;
+use bevy_ecs::query::Has;
+use bevy_ecs::system::Res;
+use bevy_ecs::{
+    entity::Entity, event::EventWriter, query::With, resource::Resource,
+    schedule::IntoSystemConfigs, system::Query,
+};
+use bevy_math::{FloatExt, Ray3d, Rect, Vec2, Vec3, Vec3Swizzles};
+use bevy_picking::Pickable;
+use bevy_picking::{
+    backend::{HitData, PointerHits},
+    pointer::{PointerId, PointerLocation},
+    PickSet,
+};
+use bevy_reflect::prelude::ReflectDefault;
+use bevy_reflect::Reflect;
+use bevy_render::camera::{Camera, OrthographicProjection, Projection};
+use bevy_render::view::ViewVisibility;
+use bevy_transform::components::GlobalTransform;
+use bevy_window::PrimaryWindow;
+
+use crate::{Text2d, TextBounds, TextLayoutInfo};
+
+/// Runtime settings for the [`Text2dPickingPlugin`].
+#[derive(Default, Resource, Reflect)]
+#[reflect(Resource, Default)]
+pub struct Text2dPickingSettings {
+    /// When set to `true` picking will only consider cameras marked with
+    /// [`Text2dPickingCamera`] and entities marked with [`Pickable`]. `false` by default.
+    ///
+    /// This setting is provided to give you fine-grained control over which cameras and entities
+    /// should be used by the picking backend at runtime.
+    pub require_markers: bool,
+}
+
+pub struct Text2dPickingPlugin;
+
+impl Plugin for Text2dPickingPlugin {
+    fn build(&self, app: &mut App) {
+        app.init_resource::<Text2dPickingSettings>()
+            .register_type::<Text2dPickingSettings>()
+            .add_systems(PreUpdate, text2d_picking.in_set(PickSet::Backend));
+    }
+}
+
+#[derive(Component)]
+pub struct Text2dPickingCamera;
+
+fn text2d_picking(
+    pointers: Query<(&PointerId, &PointerLocation)>,
+    primary_window: Query<Entity, With<PrimaryWindow>>,
+    cameras: Query<(
+        Entity,
+        &Camera,
+        &GlobalTransform,
+        &Projection,
+        Has<Text2dPickingCamera>,
+    )>,
+    text_query: Query<
+        (
+            Entity,
+            &TextLayoutInfo,
+            &TextBounds,
+            &GlobalTransform,
+            Option<&Pickable>,
+            &ViewVisibility,
+        ),
+        With<Text2d>,
+    >,
+    settings: Res<Text2dPickingSettings>,
+    mut output: EventWriter<PointerHits>,
+) {
+    let mut sorted_texts: Vec<_> = text_query
+        .iter()
+        .filter_map(|(entity, layout, bounds, transform, pickable, vis)| {
+            let marker_requirement = !settings.require_markers || pickable.is_some();
+            if !transform.affine().is_nan() && vis.get() && marker_requirement {
+                Some((entity, layout, bounds, transform, pickable))
+            } else {
+                None
+            }
+        })
+        .collect();
+
+    radsort::sort_by_key(&mut sorted_texts, |(_, _, _, t, _)| -t.translation().z);
+
+    let primary_window = primary_window.get_single().ok();
+
+    for (pointer, location) in pointers.iter().filter_map(|(pointer, pointer_location)| {
+        pointer_location.location().map(|loc| (pointer, loc))
+    }) {
+        let mut blocked = false;
+        let Some((cam_entity, camera, cam_transform, Projection::Orthographic(cam_ortho), _)) =
+            cameras
+                .iter()
+                .filter(|(_, camera, _, _, cam_can_pick)| {
+                    let marker_requirement = !settings.require_markers || *cam_can_pick;
+                    camera.is_active && marker_requirement
+                })
+                .find(|(_, camera, _, _, _)| {
+                    camera
+                        .target
+                        .normalize(primary_window)
+                        .is_some_and(|x| x == location.target)
+                })
+        else {
+            continue;
+        };
+
+        let Some((cursor_ray_world, cursor_ray_end)) =
+            rays_from_cursor_camera(location.position, camera, cam_transform, cam_ortho)
+        else {
+            continue;
+        };
+
+        let picks: Vec<(Entity, HitData)> = sorted_texts
+            .iter()
+            .filter_map(
+                |(entity, text_layout, text_bounds, text_transform, pickable)| {
+                    if blocked {
+                        return None;
+                    }
+
+                    // Transform cursor line segment to local coordinate system
+                    let relative_cursor_pos =
+                        get_relative_cursor_pos(text_transform, cursor_ray_world, cursor_ray_end)?;
+
+                    // Find target rect, check cursor is contained inside
+                    let size = Vec2::new(
+                        text_bounds.width.unwrap_or(text_layout.size.x),
+                        text_bounds.height.unwrap_or(text_layout.size.y),
+                    );
+
+                    let text_rect = Rect::from_corners(-size / 2.0, size / 2.0);
+                    if !text_rect.contains(relative_cursor_pos) {
+                        return None;
+                    }
+
+                    blocked = pickable.is_none_or(|p| p.should_block_lower);
+
+                    let hit_pos_world =
+                        text_transform.transform_point(relative_cursor_pos.extend(0.0));
+                    // Transform point from world to camera space to get the Z distance
+                    let hit_pos_cam = cam_transform
+                        .affine()
+                        .inverse()
+                        .transform_point3(hit_pos_world);
+                    // HitData requires a depth as calculated from the camera's near clipping plane
+                    let depth = -cam_ortho.near - hit_pos_cam.z;
+
+                    Some((
+                        *entity,
+                        HitData::new(
+                            cam_entity,
+                            depth,
+                            Some(hit_pos_world),
+                            Some(*text_transform.back()),
+                        ),
+                    ))
+                },
+            )
+            .collect();
+
+        let order = camera.order as f32;
+        output.send(PointerHits::new(*pointer, picks, order));
+    }
+}
+
+// TODO: find a better home for this helper
+pub fn get_relative_cursor_pos(
+    transform: &GlobalTransform,
+    world_ray: Ray3d,
+    end_ray: Vec3,
+) -> Option<Vec2> {
+    // Transform cursor line segment to target's local coordinate system
+    let world_to_target = transform.affine().inverse();
+    let cursor_start = world_to_target.transform_point3(world_ray.origin);
+    let cursor_end = world_to_target.transform_point3(end_ray);
+
+    // Find where the cursor segment intersects the plane Z=0 (which is the target's
+    // plane in local space). It may not intersect if, for example, we're
+    // viewing the target side-on
+    if cursor_start.z == cursor_end.z {
+        // Cursor ray is parallel to the text and misses it
+        return None;
+    }
+    let lerp_factor = f32::inverse_lerp(cursor_start.z, cursor_end.z, 0.0);
+    if !(0.0..=1.0).contains(&lerp_factor) {
+        // Lerp factor is out of range, meaning that while an infinite line cast by
+        // the cursor would intersect the target, the target is not between the
+        // camera's near and far planes
+        return None;
+    }
+
+    // Otherwise we can interpolate the xy of the start and end positions by the
+    // lerp factor to get the cursor position in local space
+    Some(cursor_start.lerp(cursor_end, lerp_factor).xy())
+}
+
+pub fn rays_from_cursor_camera(
+    cursor: Vec2,
+    camera: &Camera,
+    cam_transform: &GlobalTransform,
+    projection: &OrthographicProjection,
+) -> Option<(Ray3d, Vec3)> {
+    let viewport_pos = camera
+        .logical_viewport_rect()
+        .map(|v| v.min)
+        .unwrap_or_default();
+    let pos_in_viewport = cursor - viewport_pos;
+
+    let Ok(cursor_ray_world) = camera.viewport_to_world(cam_transform, pos_in_viewport) else {
+        return None;
+    };
+    let cursor_ray_len = projection.far - projection.near;
+    let cursor_ray_end = cursor_ray_world.origin + cursor_ray_world.direction * cursor_ray_len;
+    Some((cursor_ray_world, cursor_ray_end))
+}