From 6392d94eb2d84e290ebcc2c1bdd28b44bad7368b Mon Sep 17 00:00:00 2001 From: Ling Wang Date: Sat, 21 Sep 2024 21:12:56 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20basic=20test=20handle?= =?UTF-8?q?r=20for=20GUI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 7 + Cargo.toml | 1 + src/exec/gui_api.rs | 10 +- src/exec/gui_exec.rs | 216 +++++++++++++++++++++++++++ src/exec/gui_handler/basic_handle.rs | 104 +++++++++++++ src/exec/gui_handler/handler_api.rs | 24 +++ src/exec/gui_handler/mod.rs | 5 + src/exec/mod.rs | 4 +- src/exec/needle.rs | 7 + src/util/util.rs | 1 - 10 files changed, 373 insertions(+), 6 deletions(-) create mode 100644 src/exec/gui_exec.rs create mode 100644 src/exec/gui_handler/basic_handle.rs create mode 100644 src/exec/gui_handler/handler_api.rs create mode 100644 src/exec/gui_handler/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 504621f..e8e0adb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -635,6 +635,12 @@ dependencies = [ "syn", ] +[[package]] +name = "inventory" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" + [[package]] name = "io-kit-sys" version = "0.4.1" @@ -1614,6 +1620,7 @@ dependencies = [ "colored", "enigo", "image", + "inventory", "nix 0.29.0", "portable-pty", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index a1bdafe..d025548 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ image = "0.25.2" xcap = "0.0.13" enigo = "0.2.1" portable-pty = "0.8.1" +inventory = "0.3.15" [toolchain] channel = "nightly" diff --git a/src/exec/gui_api.rs b/src/exec/gui_api.rs index d94edbf..f525300 100644 --- a/src/exec/gui_api.rs +++ b/src/exec/gui_api.rs @@ -14,15 +14,17 @@ use super::needle::Needle; /// The API can used for testing, with bypass [`Screen`] operations. pub trait GuiTestApi: Screen { /// Check if the current screen is the expected screen - fn assert_screen(&mut self, needle: Needle) -> Result<(), Box>; + fn assert_screen(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box>; + /// Check and click the target position /// + /// Only Basic Needle support this, might remove in the future and move to FFI part. /// Suggest using assert_screen and click seperately, as futher consider adding relative position etc. - fn assert_screen_click(&mut self, needle: Needle) -> Result<(), Box>; + fn assert_screen_click(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box>; /// Wait until current screen changed - fn wait_screen_change(&mut self, timeout: u32) -> Result<(), Box>; + fn wait_screen_change(&mut self, timeout: u32, allow_list: Option<&[String]>) -> Result<(), Box>; /// Wait and assert the screen won't change in timeout - fn wait_still_screen(&mut self, timeout: u32) -> Result<(), Box>; + fn wait_still_screen(&mut self, timeout: u32, allow_list: Option<&[String]>) -> Result<(), Box>; } diff --git a/src/exec/gui_exec.rs b/src/exec/gui_exec.rs new file mode 100644 index 0000000..5e3cae3 --- /dev/null +++ b/src/exec/gui_exec.rs @@ -0,0 +1,216 @@ +//! Executor gor GUI +//! + +use std::{any::Any, error::Error, time::Instant}; + +use image::RgbaImage; + +use crate::{ + exec::gui_handler::{ + basic_handle::basic_handle_once, + handler_api::HandlerCollector, + }, + gui::screen::{DynScreen, Screen}, + info, + util::anybase::AnyBase, +}; + +use super::{gui_api::GuiTestApi, needle::Needle}; + +pub struct GuiTestor { + inner: DynScreen, +} + +impl GuiTestor { + pub fn build(inner: DynScreen) -> GuiTestor { + GuiTestor { inner } + } +} + +impl AnyBase for GuiTestor { + fn as_any(&self) -> &dyn Any { + self + } + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + fn into_any(self: Box) -> Box { + self + } +} + +impl Screen for GuiTestor { + fn size(&self) -> (u32, u32) { + self.inner.size() + } + fn read(&mut self) -> Result> { + self.inner.read() + } + fn move_to(&mut self, x: u32, y: u32) -> Result<(), Box> { + self.inner.move_to(x, y) + } + fn click_left(&mut self) -> Result<(), Box> { + self.inner.click_left() + } + fn click_right(&mut self) -> Result<(), Box> { + self.inner.click_right() + } + fn click_middle(&mut self) -> Result<(), Box> { + self.inner.click_middle() + } + fn scroll_up(&mut self, len: u32) -> Result<(), Box> { + self.inner.scroll_up(len) + } + fn scroll_down(&mut self, len: u32) -> Result<(), Box> { + self.inner.scroll_down(len) + } + fn write(&mut self, data: String) -> Result<(), Box> { + self.inner.write(data) + } + fn hold(&mut self, key: u16) -> Result<(), Box> { + self.inner.hold(key) + } + fn release(&mut self, key: u16) -> Result<(), Box> { + self.inner.release(key) + } +} + +struct TimeOutErr; + +impl Error for TimeOutErr {} + +impl std::fmt::Debug for TimeOutErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Timeout") + } +} + +impl std::fmt::Display for TimeOutErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Timeout") + } +} + +/// Wait for a timeout and do something +/// +/// func should return `Some(T)` if the operation is done, `None` if the operation is not done yet, and `Err(E)` if an error occurred. +fn wait_timeout_do( + timeout: u32, + func: &mut dyn FnMut() -> Result, Box>, +) -> Result> { + let begin = Instant::now(); + loop { + match func() { + Ok(Some(v)) => return Ok(v), + Ok(None) => {} + Err(e) => return Err(e), + } + if begin.elapsed().as_secs() >= timeout as u64 { + return Err(TimeOutErr.into()); + } + } +} + +impl GuiTestApi for GuiTestor { + fn assert_screen(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box> { + info!("Waiting for screen..."); + wait_timeout_do(timeout, &mut || { + let screen = self.read()?; + let mut res = None; + for handler in inventory::iter:: { + if !handler.inner.can_handle(&needle) { + continue; + } + res = Some(handler.inner.handle(&needle, &screen) || res.unwrap_or(false)); + } + match res { + Some(true) => Ok(Some(())), + Some(false) => Ok(None), + None => Err("No handler found".into()), + } + }) + } + fn assert_screen_click(&mut self, needle: &Needle, timeout: u32) -> Result<(), Box> { + self.assert_screen(&needle, timeout)?; + if !needle.is_basic() { + return Err("Only Basic Needle support this".into()); + } + let areas = match needle { + Needle::Basic(areas) => areas, + _ => unreachable!(), + }; + let screen = self.read()?; + for area in areas { + if area.click_point.is_none() { + continue; + } + if !basic_handle_once(area, &screen) { + continue; + } + let point = area.click_point.as_ref().unwrap(); + self.click_left_at(point.xpos, point.ypos)?; + } + Ok(()) + } + fn wait_screen_change( + &mut self, + timeout: u32, + allow_list: Option<&[String]>, + ) -> Result<(), Box> { + info!("Waiting for screen change..."); + let mut prev_screen = self.read()?; + wait_timeout_do(timeout, &mut || { + let screen = self.read()?; + let mut res = None; + for handler in inventory::iter:: { + if !handler.inner.can_handle_change(allow_list) { + continue; + } + res = Some( + handler.inner.handle_change(&screen, &prev_screen) || res.unwrap_or(false), + ); + } + match res { + Some(true) => Ok(Some(())), + Some(false) => { + prev_screen = screen; + Ok(None) + } + None => Err("No handler found".into()), + } + }) + } + fn wait_still_screen( + &mut self, + timeout: u32, + allow_list: Option<&[String]>, + ) -> Result<(), Box> { + info!("Waiting for still screen..."); + let mut prev_screen = self.read()?; + let changed = wait_timeout_do(timeout, &mut || { + let screen = self.read()?; + let mut res = None; + for handler in inventory::iter:: { + if !handler.inner.can_handle_change(allow_list) { + continue; + } + res = Some( + handler.inner.handle_change(&screen, &prev_screen) || res.unwrap_or(false), + ); + } + match res { + Some(true) => Ok(Some(())), + Some(false) => { + prev_screen = screen; + Ok(None) + } + None => Err("No handler found".into()), + } + }); + match changed { + Ok(_) => Err("Screen changed".into()), + Err(e) if e.to_string() == "Timeout" => Ok(()), + Err(e) => Err(e), + } + } +} diff --git a/src/exec/gui_handler/basic_handle.rs b/src/exec/gui_handler/basic_handle.rs new file mode 100644 index 0000000..d57f778 --- /dev/null +++ b/src/exec/gui_handler/basic_handle.rs @@ -0,0 +1,104 @@ +//! Basic handler for GUI. Which also means OpenQA compatible. + + +use image::RgbaImage; + +use crate::exec::{ + gui_handler::handler_api::HandlerCollector, + needle::{Area, Needle, NeedleType}, +}; + +use super::handler_api::GuiHandler; + +pub struct BasicHandler {} + +pub fn basic_handle_once(area: &Area, screen: &RgbaImage) -> bool { + let target = match &area.target { + Some(target) => target, + None => return false, + }; + if target.width() != screen.width() || target.height() != screen.height() { + return false; + } + match area.needle { + NeedleType::Match => { + let mut match_count = 0; + let total_count = area.width * area.height; + for x in 0..area.width { + for y in 0..area.height { + let x = area.x + x; + let y = area.y + y; + let pixel = screen.get_pixel(x, y); + let target_pixel = target.get_pixel(x, y); + if pixel == target_pixel { + match_count += 1; + } + } + } + let similarity = match_count as f32 / total_count as f32; + similarity >= area.match_threhold + } + NeedleType::Ocr => return false, + NeedleType::Exclude => { + let mut match_count = 0; + let total_count = screen.width() * screen.height() - area.width * area.height; + for x in 0..screen.width() { + for y in 0..screen.height() { + if x >= area.x + && x < area.x + area.width + && y >= area.y + && y < area.y + area.height + { + continue; + } + let pixel = screen.get_pixel(x, y); + let target_pixel = target.get_pixel(x, y); + if pixel == target_pixel { + match_count += 1; + } + } + } + let similarity = match_count as f32 / total_count as f32; + similarity >= area.match_threhold + } + } +} + +impl GuiHandler for BasicHandler { + fn can_handle(&self, needle: &Needle) -> bool { + needle.is_basic() + } + fn handle(&self, needle: &Needle, screen: &RgbaImage) -> bool { + let needle = match needle { + Needle::Basic(areas) => areas, + _ => return false, + }; + needle.iter().all(|area| basic_handle_once(area, screen)) + } + fn can_handle_change(&self, allow_list: Option<&[String]>) -> bool { + if let Some(allow_list) = allow_list { + allow_list.iter().any(|x| x == "basic") + } else { + true + } + } + fn handle_change(&self, screen: &RgbaImage, prev_screen: &RgbaImage) -> bool { + for x in 0..screen.width() { + for y in 0..screen.height() { + let pixel = screen.get_pixel(x, y); + let prev_pixel = prev_screen.get_pixel(x, y); + if pixel != prev_pixel { + return false; + } + } + } + true + // screen != prev_screen + } +} + +inventory::submit! { + HandlerCollector { + inner: &BasicHandler {} + } +} diff --git a/src/exec/gui_handler/handler_api.rs b/src/exec/gui_handler/handler_api.rs new file mode 100644 index 0000000..b711117 --- /dev/null +++ b/src/exec/gui_handler/handler_api.rs @@ -0,0 +1,24 @@ +//! Handler API for GUI + +use image::RgbaImage; + +use crate::exec::needle::Needle; + +pub trait GuiHandler { + /// Check if this handler can handle this type of needle + fn can_handle(&self, needle: &Needle) -> bool; + /// Handle the needle, return if the screen matches the needle + fn handle(&self, needle: &Needle, screen: &RgbaImage) -> bool; + // Check if this handler can handle screen change + fn can_handle_change(&self, allow_list: Option<&[String]>) -> bool; + /// Handle the screen change, return if the screen matches the previous screen + fn handle_change(&self, screen: &RgbaImage, prev_screen: &RgbaImage) -> bool; +} + +pub type DynGuiHandler = dyn GuiHandler + Send + Sync + 'static; + +pub struct HandlerCollector { + pub inner: &'static DynGuiHandler, +} + +inventory::collect!(HandlerCollector); diff --git a/src/exec/gui_handler/mod.rs b/src/exec/gui_handler/mod.rs new file mode 100644 index 0000000..a64edf9 --- /dev/null +++ b/src/exec/gui_handler/mod.rs @@ -0,0 +1,5 @@ +//! Handles for the GUI needle matching +//! + +pub mod handler_api; +pub mod basic_handle; \ No newline at end of file diff --git a/src/exec/mod.rs b/src/exec/mod.rs index e123d88..3306382 100644 --- a/src/exec/mod.rs +++ b/src/exec/mod.rs @@ -1,5 +1,7 @@ pub mod cli_api; pub mod cli_exec; pub mod gui_api; +pub mod needle; +pub mod gui_exec; -pub mod needle; \ No newline at end of file +pub mod gui_handler; \ No newline at end of file diff --git a/src/exec/needle.rs b/src/exec/needle.rs index ac1b388..93e4134 100644 --- a/src/exec/needle.rs +++ b/src/exec/needle.rs @@ -14,6 +14,13 @@ pub enum Needle { // In future, consider adding more needle types, // like using neural network to match the screen // to get a better flexibility + NeedleEnd, // remove will warn about unreachable pattern +} + +impl Needle { + pub fn is_basic(&self) -> bool { + matches!(self, Needle::Basic(_)) + } } /// Needle type diff --git a/src/util/util.rs b/src/util/util.rs index 1afd1b6..25f79a4 100644 --- a/src/util/util.rs +++ b/src/util/util.rs @@ -5,7 +5,6 @@ use std::{ use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use crate::info; #[macro_export] macro_rules! todo {