✨ feat: Add basic test handler for GUI

Cargo.toml

@@ -24,6 +24,7 @@ image = "0.25.2"
 xcap = "0.0.13"
 enigo = "0.2.1"
 portable-pty = "0.8.1"
+inventory = "0.3.15"
 
 [toolchain]
 channel = "nightly"

src/exec/gui_api.rs

@@ -14,15 +14,17 @@ use super::needle::Needle;
 /// The API can used for testing, with bypass [`Screen`] operations.
 pub trait GuiTestApi: Screen {
     /// Check if the current screen is the expected screen
-    fn assert_screen(&mut self, needle: Needle) -> Result<(), Box<dyn Error>>;
+    fn assert_screen(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box<dyn Error>>;
+
     /// Check and click the target position
     /// 
+    /// Only Basic Needle support this, might remove in the future and move to FFI part.
     /// Suggest using assert_screen and click seperately, as futher consider adding relative position etc.
-    fn assert_screen_click(&mut self, needle: Needle) -> Result<(), Box<dyn Error>>;
+    fn assert_screen_click(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box<dyn Error>>;
 
     /// Wait until current screen changed
-    fn wait_screen_change(&mut self, timeout: u32) -> Result<(), Box<dyn Error>>;
+    fn wait_screen_change(&mut self, timeout: u32, allow_list: Option<&[String]>) -> Result<(), Box<dyn Error>>;
 
     /// Wait and assert the screen won't change in timeout
-    fn wait_still_screen(&mut self, timeout: u32) -> Result<(), Box<dyn Error>>;
+    fn wait_still_screen(&mut self, timeout: u32, allow_list: Option<&[String]>) -> Result<(), Box<dyn Error>>;
 }

src/exec/gui_exec.rs

@@ -0,0 +1,216 @@
+//! Executor gor GUI
+//!
+
+use std::{any::Any, error::Error, time::Instant};
+
+use image::RgbaImage;
+
+use crate::{
+    exec::gui_handler::{
+        basic_handle::basic_handle_once,
+        handler_api::HandlerCollector,
+    },
+    gui::screen::{DynScreen, Screen},
+    info,
+    util::anybase::AnyBase,
+};
+
+use super::{gui_api::GuiTestApi, needle::Needle};
+
+pub struct GuiTestor {
+    inner: DynScreen,
+}
+
+impl GuiTestor {
+    pub fn build(inner: DynScreen) -> GuiTestor {
+        GuiTestor { inner }
+    }
+}
+
+impl AnyBase for GuiTestor {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+    fn into_any(self: Box<Self>) -> Box<dyn Any> {
+        self
+    }
+}
+
+impl Screen for GuiTestor {
+    fn size(&self) -> (u32, u32) {
+        self.inner.size()
+    }
+    fn read(&mut self) -> Result<RgbaImage, Box<dyn Error>> {
+        self.inner.read()
+    }
+    fn move_to(&mut self, x: u32, y: u32) -> Result<(), Box<dyn Error>> {
+        self.inner.move_to(x, y)
+    }
+    fn click_left(&mut self) -> Result<(), Box<dyn Error>> {
+        self.inner.click_left()
+    }
+    fn click_right(&mut self) -> Result<(), Box<dyn Error>> {
+        self.inner.click_right()
+    }
+    fn click_middle(&mut self) -> Result<(), Box<dyn Error>> {
+        self.inner.click_middle()
+    }
+    fn scroll_up(&mut self, len: u32) -> Result<(), Box<dyn Error>> {
+        self.inner.scroll_up(len)
+    }
+    fn scroll_down(&mut self, len: u32) -> Result<(), Box<dyn Error>> {
+        self.inner.scroll_down(len)
+    }
+    fn write(&mut self, data: String) -> Result<(), Box<dyn Error>> {
+        self.inner.write(data)
+    }
+    fn hold(&mut self, key: u16) -> Result<(), Box<dyn Error>> {
+        self.inner.hold(key)
+    }
+    fn release(&mut self, key: u16) -> Result<(), Box<dyn Error>> {
+        self.inner.release(key)
+    }
+}
+
+struct TimeOutErr;
+
+impl Error for TimeOutErr {}
+
+impl std::fmt::Debug for TimeOutErr {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Timeout")
+    }
+}
+
+impl std::fmt::Display for TimeOutErr {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Timeout")
+    }
+}
+
+/// Wait for a timeout and do something
+///
+/// func should return `Some(T)` if the operation is done, `None` if the operation is not done yet, and `Err(E)` if an error occurred.
+fn wait_timeout_do<T>(
+    timeout: u32,
+    func: &mut dyn FnMut() -> Result<Option<T>, Box<dyn Error>>,
+) -> Result<T, Box<dyn Error>> {
+    let begin = Instant::now();
+    loop {
+        match func() {
+            Ok(Some(v)) => return Ok(v),
+            Ok(None) => {}
+            Err(e) => return Err(e),
+        }
+        if begin.elapsed().as_secs() >= timeout as u64 {
+            return Err(TimeOutErr.into());
+        }
+    }
+}
+
+impl GuiTestApi for GuiTestor {
+    fn assert_screen(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box<dyn Error>> {
+        info!("Waiting for screen...");
+        wait_timeout_do(timeout, &mut || {
+            let screen = self.read()?;
+            let mut res = None;
+            for handler in inventory::iter::<HandlerCollector> {
+                if !handler.inner.can_handle(&needle) {
+                    continue;
+                }
+                res = Some(handler.inner.handle(&needle, &screen) || res.unwrap_or(false));
+            }
+            match res {
+                Some(true) => Ok(Some(())),
+                Some(false) => Ok(None),
+                None => Err("No handler found".into()),
+            }
+        })
+    }
+    fn assert_screen_click(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box<dyn Error>> {
+        self.assert_screen(&needle, timeout)?;
+        if !needle.is_basic() {
+            return Err("Only Basic Needle support this".into());
+        }
+        let areas = match needle {
+            Needle::Basic(areas) => areas,
+            _ => unreachable!(),
+        };
+        let screen = self.read()?;
+        for area in areas {
+            if area.click_point.is_none() {
+                continue;
+            }
+            if !basic_handle_once(area, &screen) {
+                continue;
+            }
+            let point = area.click_point.as_ref().unwrap();
+            self.click_left_at(point.xpos, point.ypos)?;
+        }
+        Ok(())
+    }
+    fn wait_screen_change(
+        &mut self,
+        timeout: u32,
+        allow_list: Option<&[String]>,
+    ) -> Result<(), Box<dyn Error>> {
+        info!("Waiting for screen change...");
+        let mut prev_screen = self.read()?;
+        wait_timeout_do(timeout, &mut || {
+            let screen = self.read()?;
+            let mut res = None;
+            for handler in inventory::iter::<HandlerCollector> {
+                if !handler.inner.can_handle_change(allow_list) {
+                    continue;
+                }
+                res = Some(
+                    handler.inner.handle_change(&screen, &prev_screen) || res.unwrap_or(false),
+                );
+            }
+            match res {
+                Some(true) => Ok(Some(())),
+                Some(false) => {
+                    prev_screen = screen;
+                    Ok(None)
+                }
+                None => Err("No handler found".into()),
+            }
+        })
+    }
+    fn wait_still_screen(
+        &mut self,
+        timeout: u32,
+        allow_list: Option<&[String]>,
+    ) -> Result<(), Box<dyn Error>> {
+        info!("Waiting for still screen...");
+        let mut prev_screen = self.read()?;
+        let changed = wait_timeout_do(timeout, &mut || {
+            let screen = self.read()?;
+            let mut res = None;
+            for handler in inventory::iter::<HandlerCollector> {
+                if !handler.inner.can_handle_change(allow_list) {
+                    continue;
+                }
+                res = Some(
+                    handler.inner.handle_change(&screen, &prev_screen) || res.unwrap_or(false),
+                );
+            }
+            match res {
+                Some(true) => Ok(Some(())),
+                Some(false) => {
+                    prev_screen = screen;
+                    Ok(None)
+                }
+                None => Err("No handler found".into()),
+            }
+        });
+        match changed {
+            Ok(_) => Err("Screen changed".into()),
+            Err(e) if e.to_string() == "Timeout" => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+}

src/exec/gui_handler/basic_handle.rs

@@ -0,0 +1,104 @@
+//! Basic handler for GUI. Which also means OpenQA compatible.
+
+
+use image::RgbaImage;
+
+use crate::exec::{
+    gui_handler::handler_api::HandlerCollector,
+    needle::{Area, Needle, NeedleType},
+};
+
+use super::handler_api::GuiHandler;
+
+pub struct BasicHandler {}
+
+pub fn basic_handle_once(area: &Area, screen: &RgbaImage) -> bool {
+    let target = match &area.target {
+        Some(target) => target,
+        None => return false,
+    };
+    if target.width() != screen.width() || target.height() != screen.height() {
+        return false;
+    }
+    match area.needle {
+        NeedleType::Match => {
+            let mut match_count = 0;
+            let total_count = area.width * area.height;
+            for x in 0..area.width {
+                for y in 0..area.height {
+                    let x = area.x + x;
+                    let y = area.y + y;
+                    let pixel = screen.get_pixel(x, y);
+                    let target_pixel = target.get_pixel(x, y);
+                    if pixel == target_pixel {
+                        match_count += 1;
+                    }
+                }
+            }
+            let similarity = match_count as f32 / total_count as f32;
+            similarity >= area.match_threhold
+        }
+        NeedleType::Ocr => return false,
+        NeedleType::Exclude => {
+            let mut match_count = 0;
+            let total_count = screen.width() * screen.height() - area.width * area.height;
+            for x in 0..screen.width() {
+                for y in 0..screen.height() {
+                    if x >= area.x
+                        && x < area.x + area.width
+                        && y >= area.y
+                        && y < area.y + area.height
+                    {
+                        continue;
+                    }
+                    let pixel = screen.get_pixel(x, y);
+                    let target_pixel = target.get_pixel(x, y);
+                    if pixel == target_pixel {
+                        match_count += 1;
+                    }
+                }
+            }
+            let similarity = match_count as f32 / total_count as f32;
+            similarity >= area.match_threhold
+        }
+    }
+}
+
+impl GuiHandler for BasicHandler {
+    fn can_handle(&self, needle: &Needle) -> bool {
+        needle.is_basic()
+    }
+    fn handle(&self, needle: &Needle, screen: &RgbaImage) -> bool {
+        let needle = match needle {
+            Needle::Basic(areas) => areas,
+            _ => return false,
+        };
+        needle.iter().all(|area| basic_handle_once(area, screen))
+    }
+    fn can_handle_change(&self, allow_list: Option<&[String]>) -> bool {
+        if let Some(allow_list) = allow_list {
+            allow_list.iter().any(|x| x == "basic")
+        } else {
+            true
+        }
+    }
+    fn handle_change(&self, screen: &RgbaImage, prev_screen: &RgbaImage) -> bool {
+        for x in 0..screen.width() {
+            for y in 0..screen.height() {
+                let pixel = screen.get_pixel(x, y);
+                let prev_pixel = prev_screen.get_pixel(x, y);
+                if pixel != prev_pixel {
+                    return false;
+                }
+            }
+        }
+        true
+        // screen != prev_screen
+    }
+}
+
+inventory::submit! {
+    HandlerCollector {
+        inner: &BasicHandler {}
+    }
+}

src/exec/gui_handler/handler_api.rs

@@ -0,0 +1,24 @@
+//! Handler API for GUI
+
+use image::RgbaImage;
+
+use crate::exec::needle::Needle;
+
+pub trait GuiHandler {
+    /// Check if this handler can handle this type of needle
+    fn can_handle(&self, needle: &Needle) -> bool;
+    /// Handle the needle, return if the screen matches the needle
+    fn handle(&self, needle: &Needle, screen: &RgbaImage) -> bool;
+    // Check if this handler can handle screen change
+    fn can_handle_change(&self, allow_list: Option<&[String]>) -> bool;
+    /// Handle the screen change, return if the screen matches the previous screen
+    fn handle_change(&self, screen: &RgbaImage, prev_screen: &RgbaImage) -> bool;
+}
+
+pub type DynGuiHandler = dyn GuiHandler + Send + Sync + 'static;
+
+pub struct HandlerCollector {
+    pub inner: &'static DynGuiHandler,
+}
+
+inventory::collect!(HandlerCollector);