From 505f02eace4b2eba4c19de347f733e73966b3fbd Mon Sep 17 00:00:00 2001 From: TXXT Date: Wed, 20 May 2020 15:04:07 +0800 Subject: [PATCH 01/15] init Signed-off-by: TXXT --- src/lib.rs | 395 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 277 insertions(+), 118 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fd090ce..186b41e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ //! I/O panic: //! //! ```rust -//! use fail::{fail_point, FailScenario}; +//! use fail::fail_point; //! //! fn do_fallible_work() { //! fail_point!("read-dir"); @@ -36,9 +36,9 @@ //! // ... do some work on the directory ... //! } //! -//! let scenario = FailScenario::setup(); +//! fail::setup(); //! do_fallible_work(); -//! scenario.teardown(); +//! fail::teardown(); //! println!("done"); //! ``` //! @@ -75,14 +75,20 @@ //! The previous example triggers a fail point by modifying the `FAILPOINT` //! environment variable. In practice, you'll often want to trigger fail points //! programmatically, in unit tests. -//! Fail points are global resources, and Rust tests run in parallel, -//! so tests that exercise fail points generally need to hold a lock to -//! avoid interfering with each other. This is accomplished by `FailScenario`. +//! +//! Fail points are managed by the registry which store a map of the fail points +//! names and actions. The registry is divided into local and global. +//! +//! When you don't specifically declare a registry, the global registry will be used +//! by default. You can pass the setting from environment variables to the global registry. +//! Sometimes you need different tests to use different registries and don’t want their +//! behavior to interfere with each other. You can create a local registry and then register +//! threads that need to share the same registry. //! //! Here's a basic pattern for writing unit tests tests with fail points: //! //! ```rust -//! use fail::{fail_point, FailScenario}; +//! use fail::fail_point; //! //! fn do_fallible_work() { //! fail_point!("read-dir"); @@ -93,28 +99,57 @@ //! #[test] //! #[should_panic] //! fn test_fallible_work() { -//! let scenario = FailScenario::setup(); +//! let local_registry = fail::new_fail_group(); +//! local_registry.register_current(); //! fail::cfg("read-dir", "panic").unwrap(); //! //! do_fallible_work(); //! -//! scenario.teardown(); +//! local_registry.cleanup(); //! } //! ``` //! -//! Even if a test does not itself turn on any fail points, code that it runs -//! could trigger a fail point that was configured by another thread. Because of -//! this it is a best practice to put all fail point unit tests into their own -//! binary. Here's an example of a snippet from `Cargo.toml` that creates a -//! fail-point-specific test binary: +//! It should be noted that the local registry will inherit the global registry when +//! it is created, which means that the inherited part can be shared. When you remove +//! a global fail point action from the local registry, it will affect all threads +//! using this fail point. //! -//! ```toml -//! [[test]] -//! name = "failpoints" -//! path = "tests/failpoints/mod.rs" -//! required-features = ["fail/failpoints"] +//! Here's a example to show the inheritance process： +//! +//! ```rust +//! fail::setup(); +//! fail::cfg("p1", "sleep(100)").unwrap(); +//! println!("Global registry: {:?}", fail::list()); +//! { +//! let local_registry = fail::new_fail_group(); +//! local_registry.register_current(); +//! fail::cfg("p0", "pause").unwrap(); +//! println!("Local registry: {:?}", fail::list()); +//! local_registry.cleanup(); +//! println!("Local registry: {:?}", fail::list()); +//! local_registry.deregister_current(); +//! } +//! println!("Global registry: {:?}", fail::list()); +//! ``` +//! When the example is run normally it prints out the contents of the registry used +//! at the time. +//! +//! ```sh +//! FAILPOINTS=p0=return cargo run --features fail/failpoints +//! Finished dev [unoptimized + debuginfo] target(s) in 0.01s +//! Running `target/debug/failpointtest` +//! Global registry: [("p1", "sleep(100)"), ("p0", "return")] +//! Local registry: [("p1", "sleep(100)"), ("p0", "pause")] +//! Local registry: [] +//! Global registry: [("p1", "sleep(100)"), ("p0", "return")] //! ``` //! +//! In this example, program update global registry with environment variable first. +//! Then config "p1" with "sleep(100)" in global registry because up to now, it has not +//! been bound to a local registry. After that, we create a new fail group and the +//! registry is also replaced with a local registry correspondingly. Finally, we print +//! out the global registry to show that the operations of the two registries do not +//! affect each other. //! //! ## Early return //! @@ -131,7 +166,7 @@ //! function we used earlier to return a `Result`: //! //! ```rust -//! use fail::{fail_point, FailScenario}; +//! use fail::fail_point; //! use std::io; //! //! fn do_fallible_work() -> io::Result<()> { @@ -142,9 +177,9 @@ //! } //! //! fn main() -> io::Result<()> { -//! let scenario = FailScenario::setup(); +//! fail::setup(); //! do_fallible_work()?; -//! scenario.teardown(); +//! fail::teardown(); //! println!("done"); //! Ok(()) //! } @@ -230,7 +265,7 @@ use std::env::VarError; use std::fmt::Debug; use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex, MutexGuard, RwLock, TryLockError}; +use std::sync::{Arc, Condvar, Mutex, RwLock, TryLockError}; use std::time::{Duration, Instant}; use std::{env, thread}; @@ -290,6 +325,19 @@ struct Action { count: Option, } +impl Clone for Action { + fn clone(&self) -> Self { + Action { + count: self + .count + .as_ref() + .map(|c| AtomicUsize::new(c.load(Ordering::Relaxed))), + task: self.task.clone(), + freq: self.freq, + } + } +} + impl PartialEq for Action { fn eq(&self, hs: &Action) -> bool { if self.task != hs.task || self.freq != hs.freq { @@ -421,7 +469,7 @@ impl FromStr for Action { } #[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] -#[derive(Debug)] +#[derive(Debug, Default)] struct FailPoint { pause: Mutex, pause_notifier: Condvar, @@ -429,15 +477,20 @@ struct FailPoint { actions_str: RwLock, } +impl Clone for FailPoint { + fn clone(&self) -> Self { + FailPoint { + actions: RwLock::new(self.actions.read().unwrap().clone()), + actions_str: RwLock::new(self.actions_str.read().unwrap().clone()), + ..Default::default() + } + } +} + #[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] impl FailPoint { fn new() -> FailPoint { - FailPoint { - pause: Mutex::new(false), - pause_notifier: Condvar::new(), - actions: RwLock::default(), - actions_str: RwLock::default(), - } + FailPoint::default() } fn set_actions(&self, actions_str: &str, actions: Vec) { @@ -509,98 +562,122 @@ impl FailPoint { /// Registry with failpoints configuration. type Registry = HashMap>; -#[derive(Debug, Default)] -struct FailPointRegistry { +/// Fail point registry. Threads bound to the same registry share the same +/// failpoints configuration. +#[derive(Debug, Default, Clone)] +pub struct FailPointRegistry { // TODO: remove rwlock or store *mut FailPoint - registry: RwLock, + registry: Arc>, } -lazy_static::lazy_static! { - static ref REGISTRY: FailPointRegistry = FailPointRegistry::default(); - static ref SCENARIO: Mutex<&'static FailPointRegistry> = Mutex::new(®ISTRY); +/// Generate a new failpoint registry. The new registry will inherit the +/// global failpoints configuration. +/// +/// Each thread should be bound to exact one registry. Threads bound to the +/// same registry share the same failpoints configuration. +pub fn new_fail_group() -> FailPointRegistry { + let registry = REGISTRY_GLOBAL.registry.read().unwrap(); + let mut new_registry = Registry::new(); + for (name, failpoint) in registry.iter() { + new_registry.insert(name.clone(), Arc::new(FailPoint::clone(failpoint))); + } + FailPointRegistry { + registry: Arc::new(RwLock::new(new_registry)), + } } -/// Test scenario with configured fail points. -#[derive(Debug)] -pub struct FailScenario<'a> { - scenario_guard: MutexGuard<'a, &'static FailPointRegistry>, -} +impl FailPointRegistry { + /// Register the current thread to this failpoints registry. + pub fn register_current(&self) { + let id = thread::current().id(); + REGISTRY_GROUP + .write() + .unwrap() + .insert(id, self.registry.clone()); + } -impl<'a> FailScenario<'a> { - /// Set up the system for a fail points scenario. - /// - /// Configures all fail points specified in the `FAILPOINTS` environment variable. - /// It does not otherwise change any existing fail point configuration. - /// - /// The format of `FAILPOINTS` is `failpoint=actions;...`, where - /// `failpoint` is the name of the fail point. For more information - /// about fail point actions see the [`cfg`](fn.cfg.html) function and - /// the [`fail_point`](macro.fail_point.html) macro. - /// - /// `FAILPOINTS` may configure fail points that are not actually defined. In - /// this case the configuration has no effect. - /// - /// This function should generally be called prior to running a test with fail - /// points, and afterward paired with [`teardown`](#method.teardown). - /// - /// # Panics - /// - /// Panics if an action is not formatted correctly. - pub fn setup() -> Self { - // Cleanup first, in case of previous failed/panic'ed test scenarios. - let scenario_guard = SCENARIO.lock().unwrap_or_else(|e| e.into_inner()); - let mut registry = scenario_guard.registry.write().unwrap(); - Self::cleanup(&mut registry); - - let failpoints = match env::var("FAILPOINTS") { - Ok(s) => s, - Err(VarError::NotPresent) => return Self { scenario_guard }, - Err(e) => panic!("invalid failpoints: {:?}", e), - }; - for mut cfg in failpoints.trim().split(';') { - cfg = cfg.trim(); - if cfg.is_empty() { - continue; - } - let (name, order) = partition(cfg, '='); - match order { - None => panic!("invalid failpoint: {:?}", cfg), - Some(order) => { - if let Err(e) = set(&mut registry, name.to_owned(), order) { - panic!("unable to configure failpoint \"{}\": {}", name, e); - } - } - } - } - Self { scenario_guard } + /// Deregister the current thread to this failpoints registry. + pub fn deregister_current(&self) { + let id = thread::current().id(); + REGISTRY_GROUP.write().unwrap().remove(&id); } - /// Tear down the fail point system. - /// - /// Clears the configuration of all fail points. Any paused fail - /// points will be notified before they are deactivated. - /// - /// This function should generally be called after running a test with fail points. - /// Calling `teardown` without previously calling `setup` results in a no-op. - pub fn teardown(self) { - drop(self) + /// Clean up registered fail points in this registry. + pub fn cleanup(&self) { + let mut registry = self.registry.write().unwrap(); + cleanup(&mut registry); } +} - /// Clean all registered fail points. - fn cleanup(registry: &mut std::sync::RwLockWriteGuard<'a, Registry>) { - for p in registry.values() { - // wake up all pause failpoint. - p.set_actions("", vec![]); +lazy_static::lazy_static! { + static ref REGISTRY_GROUP: RwLock>>> = Default::default(); + static ref REGISTRY_GLOBAL: FailPointRegistry = Default::default(); +} + +/// Set up the global fail points registry. +/// +/// Configures global fail points specified in the `FAILPOINTS` environment variable. +/// It does not otherwise change any existing fail point configuration. +/// +/// The format of `FAILPOINTS` is `failpoint=actions;...`, where +/// `failpoint` is the name of the fail point. For more information +/// about fail point actions see the [`cfg`](fn.cfg.html) function and +/// the [`fail_point`](macro.fail_point.html) macro. +/// +/// `FAILPOINTS` may configure fail points that are not actually defined. In +/// this case the configuration has no effect. +/// +/// This function should generally be called prior to running a test with fail +/// points, and afterward paired with [`teardown`](#method.teardown). +/// +/// # Panics +/// +/// Panics if an action is not formatted correctly. +pub fn setup() { + let mut registry = REGISTRY_GLOBAL.registry.write().unwrap(); + cleanup(&mut registry); + + let failpoints = match env::var("FAILPOINTS") { + Ok(s) => s, + Err(VarError::NotPresent) => return, + Err(e) => panic!("invalid failpoints: {:?}", e), + }; + for mut cfg in failpoints.trim().split(';') { + cfg = cfg.trim(); + if cfg.is_empty() { + continue; + } + let (name, order) = partition(cfg, '='); + match order { + None => panic!("invalid failpoint: {:?}", cfg), + Some(order) => { + if let Err(e) = set(&mut registry, name.to_owned(), order) { + panic!("unable to configure failpoint \"{}\": {}", name, e); + } + } } - registry.clear(); } } -impl<'a> Drop for FailScenario<'a> { - fn drop(&mut self) { - let mut registry = self.scenario_guard.registry.write().unwrap(); - Self::cleanup(&mut registry) +/// Tear down the global fail points registry. +/// +/// Clears the configuration of global fail points. Any paused fail +/// points will be notified before they are deactivated. +/// +/// This function should generally be called after running a test with fail points. +/// Calling `teardown` without previously calling `setup` results in a no-op. +pub fn teardown() { + let mut registry = REGISTRY_GLOBAL.registry.write().unwrap(); + cleanup(&mut registry); +} + +/// Clean all registered fail points. +fn cleanup(registry: &mut std::sync::RwLockWriteGuard) { + for p in registry.values() { + // wake up all pause failpoint. + p.set_actions("", vec![]); } + registry.clear(); } /// Returns whether code generation for failpoints is enabled. @@ -612,11 +689,22 @@ pub const fn has_failpoints() -> bool { cfg!(feature = "failpoints") } -/// Get all registered fail points. +/// Get all registered fail points in current registry. +/// +/// If current thread is not bound to any local registry. It will try to +/// get from the global registry. /// /// Return a vector of `(name, actions)` pairs. pub fn list() -> Vec<(String, String)> { - let registry = REGISTRY.registry.read().unwrap(); + let id = thread::current().id(); + let group = REGISTRY_GROUP.read().unwrap(); + + let registry = group + .get(&id) + .unwrap_or(®ISTRY_GLOBAL.registry) + .read() + .unwrap(); + registry .iter() .map(|(name, fp)| (name.to_string(), fp.actions_str.read().unwrap().clone())) @@ -625,8 +713,15 @@ pub fn list() -> Vec<(String, String)> { #[doc(hidden)] pub fn eval) -> R>(name: &str, f: F) -> Option { + let id = thread::current().id(); + let group = REGISTRY_GROUP.read().unwrap(); + let p = { - let registry = REGISTRY.registry.read().unwrap(); + let registry = group + .get(&id) + .unwrap_or(®ISTRY_GLOBAL.registry) + .read() + .unwrap(); match registry.get(name) { None => return None, Some(p) => p.clone(), @@ -635,7 +730,7 @@ pub fn eval) -> R>(name: &str, f: F) -> Option { p.eval(name).map(f) } -/// Configure the actions for a fail point at runtime. +/// Configure the actions for a fail point in current registry at runtime. /// /// Each fail point can be configured with a series of actions, specified by the /// `actions` argument. The format of `actions` is `action[->action...]`. When @@ -666,11 +761,18 @@ pub fn eval) -> R>(name: &str, f: F) -> Option { /// A call to `cfg` with a particular fail point name overwrites any existing actions for /// that fail point, including those set via the `FAILPOINTS` environment variable. pub fn cfg>(name: S, actions: &str) -> Result<(), String> { - let mut registry = REGISTRY.registry.write().unwrap(); + let id = thread::current().id(); + let group = REGISTRY_GROUP.read().unwrap(); + let mut registry = group + .get(&id) + .unwrap_or(®ISTRY_GLOBAL.registry) + .write() + .unwrap(); + set(&mut registry, name.into(), actions) } -/// Configure the actions for a fail point at runtime. +/// Configure the actions for a fail point in current registry at runtime. /// /// Each fail point can be configured by a callback. Process will call this callback function /// when it meet this fail-point. @@ -679,7 +781,14 @@ where S: Into, F: Fn() + Send + Sync + 'static, { - let mut registry = REGISTRY.registry.write().unwrap(); + let id = thread::current().id(); + let group = REGISTRY_GROUP.read().unwrap(); + let mut registry = group + .get(&id) + .unwrap_or(®ISTRY_GLOBAL.registry) + .write() + .unwrap(); + let p = registry .entry(name.into()) .or_insert_with(|| Arc::new(FailPoint::new())); @@ -691,9 +800,17 @@ where /// Remove a fail point. /// -/// If the fail point doesn't exist, nothing will happen. +/// If the local registry doesn't exist, it will try to delete the corresponding +/// action in the global registry. pub fn remove>(name: S) { - let mut registry = REGISTRY.registry.write().unwrap(); + let id = thread::current().id(); + let group = REGISTRY_GROUP.read().unwrap(); + let mut registry = group + .get(&id) + .unwrap_or(®ISTRY_GLOBAL.registry) + .write() + .unwrap(); + if let Some(p) = registry.remove(name.as_ref()) { // wake up all pause failpoint. p.set_actions("", vec![]); @@ -990,6 +1107,48 @@ mod tests { } } + #[test] + fn test_multiple_threads() { + let local_registry = new_fail_group(); + local_registry.register_current(); + let (tx, rx) = mpsc::channel(); + thread::spawn(move || { + local_registry.register_current(); + cfg("thread_point", "sleep(10)").unwrap(); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); + let l = list(); + assert!(l + .iter() + .find(|&x| x == &("thread_point".to_owned(), "sleep(10)".to_owned())) + .is_some()); + + let (tx, rx) = mpsc::channel(); + let t = thread::spawn(move || { + let local_registry = new_fail_group(); + local_registry.register_current(); + cfg("thread_point", "panic").unwrap(); + let l = list(); + assert!(l + .iter() + .find(|&x| x == &("thread_point".to_owned(), "panic".to_owned())) + .is_some()); + rx.recv().unwrap(); + local_registry.cleanup(); + let l = list(); + assert!(l.is_empty()); + }); + + tx.send(()).unwrap(); + let l = list(); + assert!(l + .iter() + .find(|&x| x == &("thread_point".to_owned(), "panic".to_owned())) + .is_none()); + t.join().unwrap(); + } + // This case should be tested as integration case, but when calling `teardown` other cases // like `test_pause` maybe also affected, so it's better keep it here. #[test] @@ -1007,7 +1166,7 @@ mod tests { "FAILPOINTS", "setup_and_teardown1=return;setup_and_teardown2=pause;", ); - let scenario = FailScenario::setup(); + setup(); assert_eq!(f1(), 1); let (tx, rx) = mpsc::channel(); @@ -1016,7 +1175,7 @@ mod tests { }); assert!(rx.recv_timeout(Duration::from_millis(500)).is_err()); - scenario.teardown(); + teardown(); assert_eq!(rx.recv_timeout(Duration::from_millis(500)).unwrap(), 0); assert_eq!(f1(), 0); } From 305eea5acb209be297ba937a2018124b75e06219 Mon Sep 17 00:00:00 2001 From: TXXT Date: Wed, 20 May 2020 15:15:03 +0800 Subject: [PATCH 02/15] fix doc method link Signed-off-by: TXXT --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 186b41e..f7cbac1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -628,7 +628,7 @@ lazy_static::lazy_static! { /// this case the configuration has no effect. /// /// This function should generally be called prior to running a test with fail -/// points, and afterward paired with [`teardown`](#method.teardown). +/// points, and afterward paired with [`teardown`](fn.teardown.html). /// /// # Panics /// From 033a21565d6801f4d515401e820a613acdb59f58 Mon Sep 17 00:00:00 2001 From: TXXT Date: Tue, 26 May 2020 08:48:49 +0000 Subject: [PATCH 03/15] update integration test Signed-off-by: TXXT --- tests/tests.rs | 152 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 122 insertions(+), 30 deletions(-) diff --git a/tests/tests.rs b/tests/tests.rs index fecb53c..7f187c1 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -7,8 +7,51 @@ use std::*; use fail::fail_point; +#[test] +#[cfg_attr(not(feature = "failpoints"), ignore)] +fn test_pause() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { + fail_point!("pause"); + }; + f(); + + fail::cfg("pause", "pause").unwrap(); + let (tx, rx) = mpsc::channel(); + // control `f()` is executed before next failpoint config + let (tx_before, rx_before) = mpsc::channel(); + let thread_registry = local_registry.clone(); + thread::spawn(move || { + thread_registry.register_current(); + // pause + tx_before.send(()).unwrap(); + tx.send(f()).unwrap(); + // woken up by new order pause, and then pause again. + tx_before.send(()).unwrap(); + tx.send(f()).unwrap(); + // woken up by remove, and then quit immediately. + tx.send(f()).unwrap(); + }); + + rx_before.recv().unwrap(); + assert!(rx.recv_timeout(Duration::from_millis(2000)).is_err()); + fail::cfg("pause", "pause").unwrap(); + rx.recv_timeout(Duration::from_millis(500)).unwrap(); + + rx_before.recv().unwrap(); + assert!(rx.recv_timeout(Duration::from_millis(2000)).is_err()); + fail::remove("pause"); + + rx.recv_timeout(Duration::from_millis(500)).unwrap(); + rx.recv_timeout(Duration::from_millis(500)).unwrap(); +} + #[test] fn test_off() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { fail_point!("off", |_| 2); 0 @@ -22,6 +65,9 @@ fn test_off() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_return() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { fail_point!("return", |s: Option| s .map_or(2, |s| s.parse().unwrap())); @@ -39,6 +85,9 @@ fn test_return() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_sleep() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { fail_point!("sleep"); }; @@ -56,6 +105,9 @@ fn test_sleep() { #[should_panic] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_panic() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { fail_point!("panic"); }; @@ -66,6 +118,9 @@ fn test_panic() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_print() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + struct LogCollector(Arc>>); impl log::Log for LogCollector { fn enabled(&self, _: &log::Metadata) -> bool { @@ -97,38 +152,11 @@ fn test_print() { assert_eq!(msg, "failpoint print executed."); } -#[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] -fn test_pause() { - let f = || { - fail_point!("pause"); - }; - f(); - - fail::cfg("pause", "pause").unwrap(); - let (tx, rx) = mpsc::channel(); - thread::spawn(move || { - // pause - tx.send(f()).unwrap(); - // woken up by new order pause, and then pause again. - tx.send(f()).unwrap(); - // woken up by remove, and then quit immediately. - tx.send(f()).unwrap(); - }); - - assert!(rx.recv_timeout(Duration::from_millis(500)).is_err()); - fail::cfg("pause", "pause").unwrap(); - rx.recv_timeout(Duration::from_millis(500)).unwrap(); - - assert!(rx.recv_timeout(Duration::from_millis(500)).is_err()); - fail::remove("pause"); - rx.recv_timeout(Duration::from_millis(500)).unwrap(); - - rx.recv_timeout(Duration::from_millis(500)).unwrap(); -} - #[test] fn test_yield() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { fail_point!("yield"); }; @@ -139,6 +167,9 @@ fn test_yield() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_callback() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f1 = || { fail_point!("cb"); }; @@ -160,6 +191,9 @@ fn test_callback() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_delay() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || fail_point!("delay"); let timer = Instant::now(); fail::cfg("delay", "delay(1000)").unwrap(); @@ -170,6 +204,9 @@ fn test_delay() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_freq_and_count() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = || { fail_point!("freq_and_count", |s: Option| s .map_or(2, |s| s.parse().unwrap())); @@ -191,6 +228,9 @@ fn test_freq_and_count() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_condition() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + let f = |_enabled| { fail_point!("condition", _enabled, |_| 2); 0 @@ -205,9 +245,61 @@ fn test_condition() { #[test] fn test_list() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + assert!(!fail::list().contains(&("list".to_string(), "off".to_string()))); fail::cfg("list", "off").unwrap(); assert!(fail::list().contains(&("list".to_string(), "off".to_string()))); fail::cfg("list", "return").unwrap(); assert!(fail::list().contains(&("list".to_string(), "return".to_string()))); } + +#[test] +fn test_multiple_threads_cleanup() { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + + let (tx, rx) = mpsc::channel(); + thread::spawn(move || { + local_registry.register_current(); + fail::cfg("thread_point", "sleep(10)").unwrap(); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); + let l = fail::list(); + assert!( + l.iter() + .find(|&x| x == &("thread_point".to_owned(), "sleep(10)".to_owned())) + .is_some() + && l.len() == 1 + ); + + let (tx, rx) = mpsc::channel(); + let t = thread::spawn(move || { + let local_registry = fail::new_fail_group(); + local_registry.register_current(); + fail::cfg("thread_point", "panic").unwrap(); + let l = fail::list(); + assert!( + l.iter() + .find(|&x| x == &("thread_point".to_owned(), "panic".to_owned())) + .is_some() + && l.len() == 1 + ); + rx.recv().unwrap(); + local_registry.cleanup(); + let l = fail::list(); + assert!(l.is_empty()); + }); + + tx.send(()).unwrap(); + let l = fail::list(); + assert!( + l.iter() + .find(|&x| x == &("thread_point".to_owned(), "sleep(10)".to_owned())) + .is_some() + && l.len() == 1 + ); + t.join().unwrap(); +} From ebb57452109743f8f522551c8d10684045c73bb2 Mon Sep 17 00:00:00 2001 From: TXXT Date: Tue, 26 May 2020 11:33:13 +0000 Subject: [PATCH 04/15] update integration test Signed-off-by: TXXT --- src/lib.rs | 44 ++++++++++++++++++++++---------------------- tests/tests.rs | 16 +++++----------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f7cbac1..ae8ec29 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -714,9 +714,9 @@ pub fn list() -> Vec<(String, String)> { #[doc(hidden)] pub fn eval) -> R>(name: &str, f: F) -> Option { let id = thread::current().id(); - let group = REGISTRY_GROUP.read().unwrap(); let p = { + let group = REGISTRY_GROUP.read().unwrap(); let registry = group .get(&id) .unwrap_or(®ISTRY_GLOBAL.registry) @@ -761,13 +761,13 @@ pub fn eval) -> R>(name: &str, f: F) -> Option { /// A call to `cfg` with a particular fail point name overwrites any existing actions for /// that fail point, including those set via the `FAILPOINTS` environment variable. pub fn cfg>(name: S, actions: &str) -> Result<(), String> { - let id = thread::current().id(); - let group = REGISTRY_GROUP.read().unwrap(); - let mut registry = group - .get(&id) - .unwrap_or(®ISTRY_GLOBAL.registry) - .write() - .unwrap(); + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + }; + + let mut registry = registry.write().unwrap(); set(&mut registry, name.into(), actions) } @@ -781,13 +781,13 @@ where S: Into, F: Fn() + Send + Sync + 'static, { - let id = thread::current().id(); - let group = REGISTRY_GROUP.read().unwrap(); - let mut registry = group - .get(&id) - .unwrap_or(®ISTRY_GLOBAL.registry) - .write() - .unwrap(); + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + }; + + let mut registry = registry.write().unwrap(); let p = registry .entry(name.into()) @@ -803,13 +803,13 @@ where /// If the local registry doesn't exist, it will try to delete the corresponding /// action in the global registry. pub fn remove>(name: S) { - let id = thread::current().id(); - let group = REGISTRY_GROUP.read().unwrap(); - let mut registry = group - .get(&id) - .unwrap_or(®ISTRY_GLOBAL.registry) - .write() - .unwrap(); + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + }; + + let mut registry = registry.write().unwrap(); if let Some(p) = registry.remove(name.as_ref()) { // wake up all pause failpoint. diff --git a/tests/tests.rs b/tests/tests.rs index 7f187c1..36784f8 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -19,32 +19,26 @@ fn test_pause() { fail::cfg("pause", "pause").unwrap(); let (tx, rx) = mpsc::channel(); - // control `f()` is executed before next failpoint config - let (tx_before, rx_before) = mpsc::channel(); let thread_registry = local_registry.clone(); thread::spawn(move || { thread_registry.register_current(); // pause - tx_before.send(()).unwrap(); tx.send(f()).unwrap(); // woken up by new order pause, and then pause again. - tx_before.send(()).unwrap(); tx.send(f()).unwrap(); // woken up by remove, and then quit immediately. tx.send(f()).unwrap(); }); - rx_before.recv().unwrap(); - assert!(rx.recv_timeout(Duration::from_millis(2000)).is_err()); + assert!(rx.recv_timeout(Duration::from_millis(100)).is_err()); fail::cfg("pause", "pause").unwrap(); - rx.recv_timeout(Duration::from_millis(500)).unwrap(); + rx.recv_timeout(Duration::from_millis(100)).unwrap(); - rx_before.recv().unwrap(); - assert!(rx.recv_timeout(Duration::from_millis(2000)).is_err()); + assert!(rx.recv_timeout(Duration::from_millis(100)).is_err()); fail::remove("pause"); - rx.recv_timeout(Duration::from_millis(500)).unwrap(); - rx.recv_timeout(Duration::from_millis(500)).unwrap(); + rx.recv_timeout(Duration::from_millis(100)).unwrap(); + rx.recv_timeout(Duration::from_millis(100)).unwrap(); } #[test] From fe2791d2b978daa66a8bb89cdf961ce315b5f337 Mon Sep 17 00:00:00 2001 From: Xintao Date: Mon, 1 Jun 2020 07:04:28 +0000 Subject: [PATCH 05/15] revert Signed-off-by: Xintao --- src/lib.rs | 48 +++--------------------------------------------- tests/tests.rs | 2 +- 2 files changed, 4 insertions(+), 46 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ae8ec29..6a8343b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -105,7 +105,7 @@ //! //! do_fallible_work(); //! -//! local_registry.cleanup(); +//! local_registry.teardown(); //! } //! ``` //! @@ -125,7 +125,7 @@ //! local_registry.register_current(); //! fail::cfg("p0", "pause").unwrap(); //! println!("Local registry: {:?}", fail::list()); -//! local_registry.cleanup(); +//! local_registry.teardown(); //! println!("Local registry: {:?}", fail::list()); //! local_registry.deregister_current(); //! } @@ -603,7 +603,7 @@ impl FailPointRegistry { } /// Clean up registered fail points in this registry. - pub fn cleanup(&self) { + pub fn teardown(&self) { let mut registry = self.registry.write().unwrap(); cleanup(&mut registry); } @@ -1107,48 +1107,6 @@ mod tests { } } - #[test] - fn test_multiple_threads() { - let local_registry = new_fail_group(); - local_registry.register_current(); - let (tx, rx) = mpsc::channel(); - thread::spawn(move || { - local_registry.register_current(); - cfg("thread_point", "sleep(10)").unwrap(); - tx.send(()).unwrap(); - }); - rx.recv().unwrap(); - let l = list(); - assert!(l - .iter() - .find(|&x| x == &("thread_point".to_owned(), "sleep(10)".to_owned())) - .is_some()); - - let (tx, rx) = mpsc::channel(); - let t = thread::spawn(move || { - let local_registry = new_fail_group(); - local_registry.register_current(); - cfg("thread_point", "panic").unwrap(); - let l = list(); - assert!(l - .iter() - .find(|&x| x == &("thread_point".to_owned(), "panic".to_owned())) - .is_some()); - rx.recv().unwrap(); - local_registry.cleanup(); - let l = list(); - assert!(l.is_empty()); - }); - - tx.send(()).unwrap(); - let l = list(); - assert!(l - .iter() - .find(|&x| x == &("thread_point".to_owned(), "panic".to_owned())) - .is_none()); - t.join().unwrap(); - } - // This case should be tested as integration case, but when calling `teardown` other cases // like `test_pause` maybe also affected, so it's better keep it here. #[test] diff --git a/tests/tests.rs b/tests/tests.rs index 36784f8..92e11c6 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -282,7 +282,7 @@ fn test_multiple_threads_cleanup() { && l.len() == 1 ); rx.recv().unwrap(); - local_registry.cleanup(); + local_registry.teardown(); let l = fail::list(); assert!(l.is_empty()); }); From 8184ded913f4d558f9e42263fd611023e4074e9b Mon Sep 17 00:00:00 2001 From: Xintao Date: Tue, 2 Jun 2020 11:58:12 +0800 Subject: [PATCH 06/15] address comments Signed-off-by: Xintao --- src/lib.rs | 112 ++++++++++++++++++++++++------------------------- tests/tests.rs | 60 +++++++++++++------------- 2 files changed, 85 insertions(+), 87 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6a8343b..0272652 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -99,7 +99,7 @@ //! #[test] //! #[should_panic] //! fn test_fallible_work() { -//! let local_registry = fail::new_fail_group(); +//! let local_registry = fail::create_registry(); //! local_registry.register_current(); //! fail::cfg("read-dir", "panic").unwrap(); //! @@ -109,19 +109,18 @@ //! } //! ``` //! -//! It should be noted that the local registry will inherit the global registry when -//! it is created, which means that the inherited part can be shared. When you remove -//! a global fail point action from the local registry, it will affect all threads -//! using this fail point. +//! It should be noted that the local registry will will overwrite the global registry +//! if you register the current thread here. This means that the current thread can only +//! use the fail points configuration of the local registry after registration. //! -//! Here's a example to show the inheritance process： +//! Here's a example to show the process： //! //! ```rust //! fail::setup(); //! fail::cfg("p1", "sleep(100)").unwrap(); //! println!("Global registry: {:?}", fail::list()); //! { -//! let local_registry = fail::new_fail_group(); +//! let local_registry = fail::create_registry(); //! local_registry.register_current(); //! fail::cfg("p0", "pause").unwrap(); //! println!("Local registry: {:?}", fail::list()); @@ -138,10 +137,10 @@ //! FAILPOINTS=p0=return cargo run --features fail/failpoints //! Finished dev [unoptimized + debuginfo] target(s) in 0.01s //! Running `target/debug/failpointtest` -//! Global registry: [("p1", "sleep(100)"), ("p0", "return")] -//! Local registry: [("p1", "sleep(100)"), ("p0", "pause")] +//! Global registry: [("p1", "sleep(100)")] +//! Local registry: [("p0", "pause")] //! Local registry: [] -//! Global registry: [("p1", "sleep(100)"), ("p0", "return")] +//! Global registry: [("p1", "sleep(100)")] //! ``` //! //! In this example, program update global registry with environment variable first. @@ -325,19 +324,6 @@ struct Action { count: Option, } -impl Clone for Action { - fn clone(&self) -> Self { - Action { - count: self - .count - .as_ref() - .map(|c| AtomicUsize::new(c.load(Ordering::Relaxed))), - task: self.task.clone(), - freq: self.freq, - } - } -} - impl PartialEq for Action { fn eq(&self, hs: &Action) -> bool { if self.task != hs.task || self.freq != hs.freq { @@ -477,16 +463,6 @@ struct FailPoint { actions_str: RwLock, } -impl Clone for FailPoint { - fn clone(&self) -> Self { - FailPoint { - actions: RwLock::new(self.actions.read().unwrap().clone()), - actions_str: RwLock::new(self.actions_str.read().unwrap().clone()), - ..Default::default() - } - } -} - #[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] impl FailPoint { fn new() -> FailPoint { @@ -575,25 +551,26 @@ pub struct FailPointRegistry { /// /// Each thread should be bound to exact one registry. Threads bound to the /// same registry share the same failpoints configuration. -pub fn new_fail_group() -> FailPointRegistry { - let registry = REGISTRY_GLOBAL.registry.read().unwrap(); - let mut new_registry = Registry::new(); - for (name, failpoint) in registry.iter() { - new_registry.insert(name.clone(), Arc::new(FailPoint::clone(failpoint))); - } +pub fn create_registry() -> FailPointRegistry { FailPointRegistry { - registry: Arc::new(RwLock::new(new_registry)), + registry: Arc::new(RwLock::new(Registry::new())), } } impl FailPointRegistry { /// Register the current thread to this failpoints registry. - pub fn register_current(&self) { + pub fn register_current(&self) -> Result<(), String> { let id = thread::current().id(); - REGISTRY_GROUP + let ret = REGISTRY_GROUP .write() .unwrap() .insert(id, self.registry.clone()); + + if ret.is_some() { + Err("current thread has been registered with one registry".to_owned()) + } else { + Ok(()) + } } /// Deregister the current thread to this failpoints registry. @@ -696,14 +673,13 @@ pub const fn has_failpoints() -> bool { /// /// Return a vector of `(name, actions)` pairs. pub fn list() -> Vec<(String, String)> { - let id = thread::current().id(); - let group = REGISTRY_GROUP.read().unwrap(); + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + }; - let registry = group - .get(&id) - .unwrap_or(®ISTRY_GLOBAL.registry) - .read() - .unwrap(); + let registry = registry.read().unwrap(); registry .iter() @@ -713,15 +689,15 @@ pub fn list() -> Vec<(String, String)> { #[doc(hidden)] pub fn eval) -> R>(name: &str, f: F) -> Option { - let id = thread::current().id(); - let p = { - let group = REGISTRY_GROUP.read().unwrap(); - let registry = group - .get(&id) - .unwrap_or(®ISTRY_GLOBAL.registry) - .read() - .unwrap(); + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + }; + + let registry = registry.read().unwrap(); + match registry.get(name) { None => return None, Some(p) => p.clone(), @@ -1125,6 +1101,28 @@ mod tests { "setup_and_teardown1=return;setup_and_teardown2=pause;", ); setup(); + + let group = create_registry(); + let handler = thread::spawn(move || { + group.register_current().unwrap(); + cfg("setup_and_teardown1", "panic").unwrap(); + cfg("setup_and_teardown2", "panic").unwrap(); + let l = list(); + assert!( + l.iter() + .find(|&x| x == &("setup_and_teardown1".to_owned(), "panic".to_owned())) + .is_some() + && l.iter() + .find(|&x| x == &("setup_and_teardown2".to_owned(), "panic".to_owned())) + .is_some() + && l.len() == 2 + ); + remove("setup_and_teardown2"); + let l = list(); + assert!(l.len() == 1); + }); + handler.join().unwrap(); + assert_eq!(f1(), 1); let (tx, rx) = mpsc::channel(); diff --git a/tests/tests.rs b/tests/tests.rs index 92e11c6..4d37cb1 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -10,8 +10,8 @@ use fail::fail_point; #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_pause() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("pause"); }; @@ -21,7 +21,7 @@ fn test_pause() { let (tx, rx) = mpsc::channel(); let thread_registry = local_registry.clone(); thread::spawn(move || { - thread_registry.register_current(); + thread_registry.register_current().unwrap(); // pause tx.send(f()).unwrap(); // woken up by new order pause, and then pause again. @@ -43,8 +43,8 @@ fn test_pause() { #[test] fn test_off() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("off", |_| 2); @@ -59,8 +59,8 @@ fn test_off() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_return() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("return", |s: Option| s @@ -79,8 +79,8 @@ fn test_return() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_sleep() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("sleep"); @@ -99,8 +99,8 @@ fn test_sleep() { #[should_panic] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_panic() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("panic"); @@ -112,8 +112,8 @@ fn test_panic() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_print() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); struct LogCollector(Arc>>); impl log::Log for LogCollector { @@ -148,8 +148,8 @@ fn test_print() { #[test] fn test_yield() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("yield"); @@ -161,8 +161,8 @@ fn test_yield() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_callback() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f1 = || { fail_point!("cb"); @@ -185,8 +185,8 @@ fn test_callback() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_delay() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || fail_point!("delay"); let timer = Instant::now(); @@ -198,8 +198,8 @@ fn test_delay() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_freq_and_count() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = || { fail_point!("freq_and_count", |s: Option| s @@ -222,8 +222,8 @@ fn test_freq_and_count() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_condition() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let f = |_enabled| { fail_point!("condition", _enabled, |_| 2); @@ -239,8 +239,8 @@ fn test_condition() { #[test] fn test_list() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); assert!(!fail::list().contains(&("list".to_string(), "off".to_string()))); fail::cfg("list", "off").unwrap(); @@ -251,12 +251,12 @@ fn test_list() { #[test] fn test_multiple_threads_cleanup() { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); let (tx, rx) = mpsc::channel(); thread::spawn(move || { - local_registry.register_current(); + local_registry.register_current().unwrap(); fail::cfg("thread_point", "sleep(10)").unwrap(); tx.send(()).unwrap(); }); @@ -271,8 +271,8 @@ fn test_multiple_threads_cleanup() { let (tx, rx) = mpsc::channel(); let t = thread::spawn(move || { - let local_registry = fail::new_fail_group(); - local_registry.register_current(); + let local_registry = fail::create_registry(); + local_registry.register_current().unwrap(); fail::cfg("thread_point", "panic").unwrap(); let l = fail::list(); assert!( From 18eafb7e9a6efbb81dd7dfa199b5a223d624b5f4 Mon Sep 17 00:00:00 2001 From: Xintao Date: Wed, 3 Jun 2020 10:59:08 +0800 Subject: [PATCH 07/15] define create_registry() as an associated function Signed-off-by: Xintao --- src/lib.rs | 27 +++++++++++++-------------- tests/tests.rs | 28 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0272652..ed7cc58 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -99,7 +99,7 @@ //! #[test] //! #[should_panic] //! fn test_fallible_work() { -//! let local_registry = fail::create_registry(); +//! let local_registry = fail::FailPointRegistry::new(); //! local_registry.register_current(); //! fail::cfg("read-dir", "panic").unwrap(); //! @@ -120,7 +120,7 @@ //! fail::cfg("p1", "sleep(100)").unwrap(); //! println!("Global registry: {:?}", fail::list()); //! { -//! let local_registry = fail::create_registry(); +//! let local_registry = fail::FailPointRegistry::new(); //! local_registry.register_current(); //! fail::cfg("p0", "pause").unwrap(); //! println!("Local registry: {:?}", fail::list()); @@ -546,18 +546,17 @@ pub struct FailPointRegistry { registry: Arc>, } -/// Generate a new failpoint registry. The new registry will inherit the -/// global failpoints configuration. -/// -/// Each thread should be bound to exact one registry. Threads bound to the -/// same registry share the same failpoints configuration. -pub fn create_registry() -> FailPointRegistry { - FailPointRegistry { - registry: Arc::new(RwLock::new(Registry::new())), - } -} - impl FailPointRegistry { + /// Generate a new failpoint registry. The new registry will inherit the + /// global failpoints configuration. + /// + /// Each thread should be bound to exact one registry. Threads bound to the + /// same registry share the same failpoints configuration. + pub fn new() -> Self { + FailPointRegistry { + registry: Arc::new(RwLock::new(Registry::new())), + } + } /// Register the current thread to this failpoints registry. pub fn register_current(&self) -> Result<(), String> { let id = thread::current().id(); @@ -1102,7 +1101,7 @@ mod tests { ); setup(); - let group = create_registry(); + let group = FailPointRegistry::new(); let handler = thread::spawn(move || { group.register_current().unwrap(); cfg("setup_and_teardown1", "panic").unwrap(); diff --git a/tests/tests.rs b/tests/tests.rs index 4d37cb1..2cee459 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -10,7 +10,7 @@ use fail::fail_point; #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_pause() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { fail_point!("pause"); @@ -43,7 +43,7 @@ fn test_pause() { #[test] fn test_off() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { @@ -59,7 +59,7 @@ fn test_off() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_return() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { @@ -79,7 +79,7 @@ fn test_return() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_sleep() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { @@ -99,7 +99,7 @@ fn test_sleep() { #[should_panic] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_panic() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { @@ -112,7 +112,7 @@ fn test_panic() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_print() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); struct LogCollector(Arc>>); @@ -148,7 +148,7 @@ fn test_print() { #[test] fn test_yield() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { @@ -161,7 +161,7 @@ fn test_yield() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_callback() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f1 = || { @@ -185,7 +185,7 @@ fn test_callback() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_delay() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || fail_point!("delay"); @@ -198,7 +198,7 @@ fn test_delay() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_freq_and_count() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = || { @@ -222,7 +222,7 @@ fn test_freq_and_count() { #[test] #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_condition() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let f = |_enabled| { @@ -239,7 +239,7 @@ fn test_condition() { #[test] fn test_list() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); assert!(!fail::list().contains(&("list".to_string(), "off".to_string()))); @@ -251,7 +251,7 @@ fn test_list() { #[test] fn test_multiple_threads_cleanup() { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); let (tx, rx) = mpsc::channel(); @@ -271,7 +271,7 @@ fn test_multiple_threads_cleanup() { let (tx, rx) = mpsc::channel(); let t = thread::spawn(move || { - let local_registry = fail::create_registry(); + let local_registry = fail::FailPointRegistry::new(); local_registry.register_current().unwrap(); fail::cfg("thread_point", "panic").unwrap(); let l = fail::list(); From d473e38e658783eec975cdbb6f45a8aa768b70b5 Mon Sep 17 00:00:00 2001 From: Xintao Date: Wed, 3 Jun 2020 13:32:03 +0800 Subject: [PATCH 08/15] revert FailScenario Signed-off-by: Xintao --- src/lib.rs | 140 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 59 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ed7cc58..63dcf32 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ //! I/O panic: //! //! ```rust -//! use fail::fail_point; +//! use fail::{fail_point, FailScenario}; //! //! fn do_fallible_work() { //! fail_point!("read-dir"); @@ -36,9 +36,9 @@ //! // ... do some work on the directory ... //! } //! -//! fail::setup(); +//! let scenario = FailScenario::setup(); //! do_fallible_work(); -//! fail::teardown(); +//! scenario.teardown(); //! println!("done"); //! ``` //! @@ -83,7 +83,8 @@ //! by default. You can pass the setting from environment variables to the global registry. //! Sometimes you need different tests to use different registries and don’t want their //! behavior to interfere with each other. You can create a local registry and then register -//! threads that need to share the same registry. +//! threads that need to share the same registry. Or you can still use `FailScenario` to +//! sequentially configure different global registry. //! //! Here's a basic pattern for writing unit tests tests with fail points: //! @@ -116,7 +117,9 @@ //! Here's a example to show the process： //! //! ```rust -//! fail::setup(); +//! use fail::FailScenario; +//! +//! let _scenario = FailScenario::setup(); //! fail::cfg("p1", "sleep(100)").unwrap(); //! println!("Global registry: {:?}", fail::list()); //! { @@ -165,7 +168,7 @@ //! function we used earlier to return a `Result`: //! //! ```rust -//! use fail::fail_point; +//! use fail::{fail_point, FailScenario}; //! use std::io; //! //! fn do_fallible_work() -> io::Result<()> { @@ -176,9 +179,9 @@ //! } //! //! fn main() -> io::Result<()> { -//! fail::setup(); +//! let scenario = FailScenario::setup(); //! do_fallible_work()?; -//! fail::teardown(); +//! scenario.teardown(); //! println!("done"); //! Ok(()) //! } @@ -264,6 +267,7 @@ use std::env::VarError; use std::fmt::Debug; use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::MutexGuard; use std::sync::{Arc, Condvar, Mutex, RwLock, TryLockError}; use std::time::{Duration, Instant}; use std::{env, thread}; @@ -588,63 +592,81 @@ impl FailPointRegistry { lazy_static::lazy_static! { static ref REGISTRY_GROUP: RwLock>>> = Default::default(); static ref REGISTRY_GLOBAL: FailPointRegistry = Default::default(); + static ref SCENARIO: Mutex<&'static FailPointRegistry> = Mutex::new(®ISTRY_GLOBAL); } -/// Set up the global fail points registry. -/// -/// Configures global fail points specified in the `FAILPOINTS` environment variable. -/// It does not otherwise change any existing fail point configuration. -/// -/// The format of `FAILPOINTS` is `failpoint=actions;...`, where -/// `failpoint` is the name of the fail point. For more information -/// about fail point actions see the [`cfg`](fn.cfg.html) function and -/// the [`fail_point`](macro.fail_point.html) macro. -/// -/// `FAILPOINTS` may configure fail points that are not actually defined. In -/// this case the configuration has no effect. -/// -/// This function should generally be called prior to running a test with fail -/// points, and afterward paired with [`teardown`](fn.teardown.html). -/// -/// # Panics -/// -/// Panics if an action is not formatted correctly. -pub fn setup() { - let mut registry = REGISTRY_GLOBAL.registry.write().unwrap(); - cleanup(&mut registry); - - let failpoints = match env::var("FAILPOINTS") { - Ok(s) => s, - Err(VarError::NotPresent) => return, - Err(e) => panic!("invalid failpoints: {:?}", e), - }; - for mut cfg in failpoints.trim().split(';') { - cfg = cfg.trim(); - if cfg.is_empty() { - continue; - } - let (name, order) = partition(cfg, '='); - match order { - None => panic!("invalid failpoint: {:?}", cfg), - Some(order) => { - if let Err(e) = set(&mut registry, name.to_owned(), order) { - panic!("unable to configure failpoint \"{}\": {}", name, e); +/// Test scenario with configured fail points. +#[derive(Debug)] +pub struct FailScenario<'a> { + scenario_guard: MutexGuard<'a, &'static FailPointRegistry>, +} + +impl<'a> FailScenario<'a> { + /// Set up the system for a fail points scenario. + /// + /// Configures global fail points specified in the `FAILPOINTS` environment variable. + /// It does not otherwise change any existing fail point configuration. + /// + /// The format of `FAILPOINTS` is `failpoint=actions;...`, where + /// `failpoint` is the name of the fail point. For more information + /// about fail point actions see the [`cfg`](fn.cfg.html) function and + /// the [`fail_point`](macro.fail_point.html) macro. + /// + /// `FAILPOINTS` may configure fail points that are not actually defined. In + /// this case the configuration has no effect. + /// + /// This function should generally be called prior to running a test with fail + /// points, and afterward paired with [`teardown`](#method.teardown). + /// + /// # Panics + /// + /// Panics if an action is not formatted correctly. + pub fn setup() -> Self { + // Cleanup first, in case of previous failed/panic'ed test scenarios. + let scenario_guard = SCENARIO.lock().unwrap_or_else(|e| e.into_inner()); + let mut registry = scenario_guard.registry.write().unwrap(); + cleanup(&mut registry); + + let failpoints = match env::var("FAILPOINTS") { + Ok(s) => s, + Err(VarError::NotPresent) => return Self { scenario_guard }, + Err(e) => panic!("invalid failpoints: {:?}", e), + }; + for mut cfg in failpoints.trim().split(';') { + cfg = cfg.trim(); + if cfg.is_empty() { + continue; + } + let (name, order) = partition(cfg, '='); + match order { + None => panic!("invalid failpoint: {:?}", cfg), + Some(order) => { + if let Err(e) = set(&mut registry, name.to_owned(), order) { + panic!("unable to configure failpoint \"{}\": {}", name, e); + } } } } + Self { scenario_guard } + } + + /// Tear down the global fail points registry. + /// + /// Clears the configuration of global fail points. Any paused fail + /// points will be notified before they are deactivated. + /// + /// This function should generally be called after running a test with fail points. + /// Calling `teardown` without previously calling `setup` results in a no-op. + pub fn teardown(self) { + drop(self); } } -/// Tear down the global fail points registry. -/// -/// Clears the configuration of global fail points. Any paused fail -/// points will be notified before they are deactivated. -/// -/// This function should generally be called after running a test with fail points. -/// Calling `teardown` without previously calling `setup` results in a no-op. -pub fn teardown() { - let mut registry = REGISTRY_GLOBAL.registry.write().unwrap(); - cleanup(&mut registry); +impl<'a> Drop for FailScenario<'a> { + fn drop(&mut self) { + let mut registry = self.scenario_guard.registry.write().unwrap(); + cleanup(&mut registry); + } } /// Clean all registered fail points. @@ -1099,7 +1121,7 @@ mod tests { "FAILPOINTS", "setup_and_teardown1=return;setup_and_teardown2=pause;", ); - setup(); + let scenario = FailScenario::setup(); let group = FailPointRegistry::new(); let handler = thread::spawn(move || { @@ -1130,7 +1152,7 @@ mod tests { }); assert!(rx.recv_timeout(Duration::from_millis(500)).is_err()); - teardown(); + scenario.teardown(); assert_eq!(rx.recv_timeout(Duration::from_millis(500)).unwrap(), 0); assert_eq!(f1(), 0); } From 04baffbeb07a088a3b28d44ff688c22c29ced575 Mon Sep 17 00:00:00 2001 From: Xintao Date: Mon, 13 Jul 2020 16:15:53 +0800 Subject: [PATCH 09/15] export current_registry Signed-off-by: Xintao --- src/lib.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 63dcf32..cbe167d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -561,6 +561,18 @@ impl FailPointRegistry { registry: Arc::new(RwLock::new(Registry::new())), } } + + /// Returns the failpoint registry to which the current thread is bound. + pub fn current_registry() -> Self { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + }; + + FailPointRegistry { registry } + } + /// Register the current thread to this failpoints registry. pub fn register_current(&self) -> Result<(), String> { let id = thread::current().id(); From b2dd693dfed4dde86d2482201222adae6653c701 Mon Sep 17 00:00:00 2001 From: Xintao Date: Tue, 14 Jul 2020 12:32:53 +0800 Subject: [PATCH 10/15] The premise of using registry is to enable failpoints Signed-off-by: Xintao --- src/lib.rs | 98 ++++++++++++++++++++++++++++++++------------------ tests/tests.rs | 36 ++++++++++--------- 2 files changed, 84 insertions(+), 50 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index cbe167d..a6739c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -547,7 +547,7 @@ type Registry = HashMap>; #[derive(Debug, Default, Clone)] pub struct FailPointRegistry { // TODO: remove rwlock or store *mut FailPoint - registry: Arc>, + registry: Option>>, } impl FailPointRegistry { @@ -558,52 +558,61 @@ impl FailPointRegistry { /// same registry share the same failpoints configuration. pub fn new() -> Self { FailPointRegistry { - registry: Arc::new(RwLock::new(Registry::new())), + registry: Some(Arc::new(RwLock::new(Registry::new()))), } } - /// Returns the failpoint registry to which the current thread is bound. + /// Returns the failpoint registry to which the current thread is bound. If + /// `failpoints` feature is not enabled, the internal registry is none. pub fn current_registry() -> Self { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() - }; - - FailPointRegistry { registry } + if cfg!(feature = "failpoints") { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group + .get(&id) + .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() + }; + FailPointRegistry { + registry: Some(registry), + } + } else { + FailPointRegistry { registry: None } + } } /// Register the current thread to this failpoints registry. - pub fn register_current(&self) -> Result<(), String> { - let id = thread::current().id(); - let ret = REGISTRY_GROUP - .write() - .unwrap() - .insert(id, self.registry.clone()); - - if ret.is_some() { - Err("current thread has been registered with one registry".to_owned()) - } else { - Ok(()) + pub fn register_current(&self) { + if cfg!(feature = "failpoints") { + let id = thread::current().id(); + REGISTRY_GROUP + .write() + .unwrap() + .insert(id, self.registry.clone().unwrap()); } } /// Deregister the current thread to this failpoints registry. pub fn deregister_current(&self) { - let id = thread::current().id(); - REGISTRY_GROUP.write().unwrap().remove(&id); + if cfg!(feature = "failpoints") { + let id = thread::current().id(); + REGISTRY_GROUP.write().unwrap().remove(&id); + } } /// Clean up registered fail points in this registry. pub fn teardown(&self) { - let mut registry = self.registry.write().unwrap(); - cleanup(&mut registry); + if cfg!(feature = "failpoints") { + let mut registry = self.registry.as_ref().unwrap().write().unwrap(); + cleanup(&mut registry); + } } } lazy_static::lazy_static! { static ref REGISTRY_GROUP: RwLock>>> = Default::default(); - static ref REGISTRY_GLOBAL: FailPointRegistry = Default::default(); + static ref REGISTRY_GLOBAL: FailPointRegistry = FailPointRegistry { registry: Some(Default::default()) }; static ref SCENARIO: Mutex<&'static FailPointRegistry> = Mutex::new(®ISTRY_GLOBAL); } @@ -636,7 +645,7 @@ impl<'a> FailScenario<'a> { pub fn setup() -> Self { // Cleanup first, in case of previous failed/panic'ed test scenarios. let scenario_guard = SCENARIO.lock().unwrap_or_else(|e| e.into_inner()); - let mut registry = scenario_guard.registry.write().unwrap(); + let mut registry = scenario_guard.registry.as_ref().unwrap().write().unwrap(); cleanup(&mut registry); let failpoints = match env::var("FAILPOINTS") { @@ -676,7 +685,13 @@ impl<'a> FailScenario<'a> { impl<'a> Drop for FailScenario<'a> { fn drop(&mut self) { - let mut registry = self.scenario_guard.registry.write().unwrap(); + let mut registry = self + .scenario_guard + .registry + .as_ref() + .unwrap() + .write() + .unwrap(); cleanup(&mut registry); } } @@ -709,7 +724,10 @@ pub fn list() -> Vec<(String, String)> { let registry = { let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + group + .get(&id) + .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() }; let registry = registry.read().unwrap(); @@ -726,7 +744,10 @@ pub fn eval) -> R>(name: &str, f: F) -> Option { let registry = { let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + group + .get(&id) + .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() }; let registry = registry.read().unwrap(); @@ -773,7 +794,10 @@ pub fn cfg>(name: S, actions: &str) -> Result<(), String> { let registry = { let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + group + .get(&id) + .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() }; let mut registry = registry.write().unwrap(); @@ -793,7 +817,10 @@ where let registry = { let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + group + .get(&id) + .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() }; let mut registry = registry.write().unwrap(); @@ -815,7 +842,10 @@ pub fn remove>(name: S) { let registry = { let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - group.get(&id).unwrap_or(®ISTRY_GLOBAL.registry).clone() + group + .get(&id) + .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() }; let mut registry = registry.write().unwrap(); @@ -1137,7 +1167,7 @@ mod tests { let group = FailPointRegistry::new(); let handler = thread::spawn(move || { - group.register_current().unwrap(); + group.register_current(); cfg("setup_and_teardown1", "panic").unwrap(); cfg("setup_and_teardown2", "panic").unwrap(); let l = list(); diff --git a/tests/tests.rs b/tests/tests.rs index 2cee459..98d682b 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -11,7 +11,7 @@ use fail::fail_point; #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_pause() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("pause"); }; @@ -21,7 +21,7 @@ fn test_pause() { let (tx, rx) = mpsc::channel(); let thread_registry = local_registry.clone(); thread::spawn(move || { - thread_registry.register_current().unwrap(); + thread_registry.register_current(); // pause tx.send(f()).unwrap(); // woken up by new order pause, and then pause again. @@ -42,9 +42,10 @@ fn test_pause() { } #[test] +#[cfg_attr(not(feature = "failpoints"), ignore)] fn test_off() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("off", |_| 2); @@ -60,7 +61,7 @@ fn test_off() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_return() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("return", |s: Option| s @@ -80,7 +81,7 @@ fn test_return() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_sleep() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("sleep"); @@ -100,7 +101,7 @@ fn test_sleep() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_panic() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("panic"); @@ -113,7 +114,7 @@ fn test_panic() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_print() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); struct LogCollector(Arc>>); impl log::Log for LogCollector { @@ -147,9 +148,10 @@ fn test_print() { } #[test] +#[cfg_attr(not(feature = "failpoints"), ignore)] fn test_yield() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("yield"); @@ -162,7 +164,7 @@ fn test_yield() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_callback() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f1 = || { fail_point!("cb"); @@ -186,7 +188,7 @@ fn test_callback() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_delay() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || fail_point!("delay"); let timer = Instant::now(); @@ -199,7 +201,7 @@ fn test_delay() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_freq_and_count() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = || { fail_point!("freq_and_count", |s: Option| s @@ -223,7 +225,7 @@ fn test_freq_and_count() { #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_condition() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let f = |_enabled| { fail_point!("condition", _enabled, |_| 2); @@ -238,9 +240,10 @@ fn test_condition() { } #[test] +#[cfg_attr(not(feature = "failpoints"), ignore)] fn test_list() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); assert!(!fail::list().contains(&("list".to_string(), "off".to_string()))); fail::cfg("list", "off").unwrap(); @@ -250,13 +253,14 @@ fn test_list() { } #[test] +#[cfg_attr(not(feature = "failpoints"), ignore)] fn test_multiple_threads_cleanup() { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); let (tx, rx) = mpsc::channel(); thread::spawn(move || { - local_registry.register_current().unwrap(); + local_registry.register_current(); fail::cfg("thread_point", "sleep(10)").unwrap(); tx.send(()).unwrap(); }); @@ -272,7 +276,7 @@ fn test_multiple_threads_cleanup() { let (tx, rx) = mpsc::channel(); let t = thread::spawn(move || { let local_registry = fail::FailPointRegistry::new(); - local_registry.register_current().unwrap(); + local_registry.register_current(); fail::cfg("thread_point", "panic").unwrap(); let l = fail::list(); assert!( From 0421da8aaf3b5ec1ba0266707ae484491b7e99a0 Mon Sep 17 00:00:00 2001 From: Xintao Date: Tue, 14 Jul 2020 20:03:17 +0800 Subject: [PATCH 11/15] ensure failpoints is enabled using fail::cfg Signed-off-by: Xintao --- src/lib.rs | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a6739c2..61cd4ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -102,7 +102,7 @@ //! fn test_fallible_work() { //! let local_registry = fail::FailPointRegistry::new(); //! local_registry.register_current(); -//! fail::cfg("read-dir", "panic").unwrap(); +//! fail::cfg("read-dir", "panic"); //! //! do_fallible_work(); //! @@ -120,12 +120,12 @@ //! use fail::FailScenario; //! //! let _scenario = FailScenario::setup(); -//! fail::cfg("p1", "sleep(100)").unwrap(); +//! fail::cfg("p1", "sleep(100)"); //! println!("Global registry: {:?}", fail::list()); //! { //! let local_registry = fail::FailPointRegistry::new(); //! local_registry.register_current(); -//! fail::cfg("p0", "pause").unwrap(); +//! fail::cfg("p0", "pause"); //! println!("Local registry: {:?}", fail::list()); //! local_registry.teardown(); //! println!("Local registry: {:?}", fail::list()); @@ -571,7 +571,7 @@ impl FailPointRegistry { let id = thread::current().id(); group .get(&id) - .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) .clone() }; FailPointRegistry { @@ -726,7 +726,7 @@ pub fn list() -> Vec<(String, String)> { let id = thread::current().id(); group .get(&id) - .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) .clone() }; @@ -746,7 +746,7 @@ pub fn eval) -> R>(name: &str, f: F) -> Option { let id = thread::current().id(); group .get(&id) - .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) .clone() }; @@ -791,18 +791,22 @@ pub fn eval) -> R>(name: &str, f: F) -> Option { /// A call to `cfg` with a particular fail point name overwrites any existing actions for /// that fail point, including those set via the `FAILPOINTS` environment variable. pub fn cfg>(name: S, actions: &str) -> Result<(), String> { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) - .clone() - }; + if cfg!(feature = "failpoints") { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group + .get(&id) + .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .clone() + }; - let mut registry = registry.write().unwrap(); + let mut registry = registry.write().unwrap(); - set(&mut registry, name.into(), actions) + set(&mut registry, name.into(), actions) + } else { + Err("failpoints is not enabled".to_owned()) + } } /// Configure the actions for a fail point in current registry at runtime. @@ -819,7 +823,7 @@ where let id = thread::current().id(); group .get(&id) - .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) .clone() }; @@ -844,7 +848,7 @@ pub fn remove>(name: S) { let id = thread::current().id(); group .get(&id) - .unwrap_or(REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) .clone() }; From db8079171af75c9d455ac0bc5d93ffc72a5d3012 Mon Sep 17 00:00:00 2001 From: Xintao Date: Tue, 28 Jul 2020 15:38:48 +0800 Subject: [PATCH 12/15] update Signed-off-by: Xintao --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 61cd4ef..d9b4675 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -129,7 +129,7 @@ //! println!("Local registry: {:?}", fail::list()); //! local_registry.teardown(); //! println!("Local registry: {:?}", fail::list()); -//! local_registry.deregister_current(); +//! fail::FailPointRegistry::deregister_current(); //! } //! println!("Global registry: {:?}", fail::list()); //! ``` @@ -594,7 +594,7 @@ impl FailPointRegistry { } /// Deregister the current thread to this failpoints registry. - pub fn deregister_current(&self) { + pub fn deregister_current() { if cfg!(feature = "failpoints") { let id = thread::current().id(); REGISTRY_GROUP.write().unwrap().remove(&id); From 23df2b98e7e81b5c514d834986b91dcf20348da7 Mon Sep 17 00:00:00 2001 From: Xintao Date: Mon, 3 Aug 2020 15:01:29 +0800 Subject: [PATCH 13/15] address comments and run doc example again to update result Signed-off-by: Xintao --- src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d9b4675..90ff159 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -76,10 +76,10 @@ //! environment variable. In practice, you'll often want to trigger fail points //! programmatically, in unit tests. //! -//! Fail points are managed by the registry which store a map of the fail points +//! Fail points are managed by the registry which stores a map of the fail points //! names and actions. The registry is divided into local and global. //! -//! When you don't specifically declare a registry, the global registry will be used +//! When you don't declare a registry explicitly, the global registry will be used //! by default. You can pass the setting from environment variables to the global registry. //! Sometimes you need different tests to use different registries and don’t want their //! behavior to interfere with each other. You can create a local registry and then register @@ -110,7 +110,7 @@ //! } //! ``` //! -//! It should be noted that the local registry will will overwrite the global registry +//! It should be noted that the local registry will overwrite the global registry //! if you register the current thread here. This means that the current thread can only //! use the fail points configuration of the local registry after registration. //! @@ -133,17 +133,17 @@ //! } //! println!("Global registry: {:?}", fail::list()); //! ``` -//! When the example is run normally it prints out the contents of the registry used +//! The example will print out the contents of the registry used //! at the time. //! //! ```sh //! FAILPOINTS=p0=return cargo run --features fail/failpoints //! Finished dev [unoptimized + debuginfo] target(s) in 0.01s //! Running `target/debug/failpointtest` -//! Global registry: [("p1", "sleep(100)")] +//! Global registry: [("p0", "return"), ("p1", "sleep(100)")] //! Local registry: [("p0", "pause")] //! Local registry: [] -//! Global registry: [("p1", "sleep(100)")] +//! Global registry: [("p0", "return"), ("p1", "sleep(100)")] //! ``` //! //! In this example, program update global registry with environment variable first. From 2481fa29e360afb75e12e7ff27a5ba5aa43e292a Mon Sep 17 00:00:00 2001 From: Xintao Date: Tue, 4 Aug 2020 14:23:16 +0800 Subject: [PATCH 14/15] reorganize the code Signed-off-by: Xintao --- .github/workflows/ci.yml | 15 +- src/lib.rs | 1255 +++++++++++++++++++------------------- tests/tests.rs | 28 +- 3 files changed, 658 insertions(+), 640 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b4b64e..d941802 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ jobs: - uses: actions/checkout@v2 - run: cargo fmt -- --check - run: cargo clippy -- -D clippy::all - - run: cargo test --all -- --nocapture + - run: cargo clippy --all-features -- -D clippy::all + - run: cargo test --all --all-features -- --ignored --nocapture - run: cargo test --all --all-features -- --nocapture - - run: cargo bench --all -- --test - run: cargo bench --all --all-features -- --test Linux-Nightly: @@ -31,9 +31,8 @@ jobs: steps: - uses: actions/checkout@v2 - run: rustup default nightly - - run: cargo test --all -- --nocapture + - run: cargo test --all --all-features -- --ignored --nocapture - run: cargo test --all --all-features -- --nocapture - - run: cargo bench --all -- --test - run: cargo bench --all --all-features -- --test Mac-Stable: @@ -41,9 +40,8 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v2 - - run: cargo test --all -- --nocapture + - run: cargo test --all --all-features -- --ignored --nocapture - run: cargo test --all --all-features -- --nocapture - - run: cargo bench --all -- --test - run: cargo bench --all --all-features -- --test Win-Stable: @@ -51,7 +49,6 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v2 - - run: cargo test --all -- --nocapture + - run: cargo test --all --all-features -- --ignored --nocapture - run: cargo test --all --all-features -- --nocapture - - run: cargo bench --all -- --test - - run: cargo bench --all --all-features -- --test \ No newline at end of file + - run: cargo bench --all --all-features -- --test diff --git a/src/lib.rs b/src/lib.rs index 90ff159..d8cf185 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,7 +27,7 @@ //! As an example, here's a simple program that uses a fail point to simulate an //! I/O panic: //! -//! ```rust +//! ```ignore //! use fail::{fail_point, FailScenario}; //! //! fn do_fallible_work() { @@ -88,7 +88,7 @@ //! //! Here's a basic pattern for writing unit tests tests with fail points: //! -//! ```rust +//! ```ignore //! use fail::fail_point; //! //! fn do_fallible_work() { @@ -116,7 +116,7 @@ //! //! Here's a example to show the process： //! -//! ```rust +//! ```ignore //! use fail::FailScenario; //! //! let _scenario = FailScenario::setup(); @@ -167,7 +167,7 @@ //! `fail_point!` macro. To illustrate this, let's modify the `do_fallible_work` //! function we used earlier to return a `Result`: //! -//! ```rust +//! ```ignore //! use fail::{fail_point, FailScenario}; //! use std::io; //! @@ -211,7 +211,7 @@ //! //! Here's a variation that does so: //! -//! ```rust +//! ```ignore //! # use std::io; //! fn do_fallible_work() -> io::Result<()> { //! fail::fail_point!("read-dir", |_| { @@ -262,719 +262,741 @@ #![deny(missing_docs, missing_debug_implementations)] -use std::collections::HashMap; -use std::env::VarError; -use std::fmt::Debug; -use std::str::FromStr; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::MutexGuard; -use std::sync::{Arc, Condvar, Mutex, RwLock, TryLockError}; -use std::time::{Duration, Instant}; -use std::{env, thread}; - -#[derive(Clone)] -struct SyncCallback(Arc); - -impl Debug for SyncCallback { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("SyncCallback()") - } -} - -impl PartialEq for SyncCallback { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.0, &other.0) - } -} +#[cfg(not(feature = "failpoints"))] +pub use disable_failpoint::*; +#[cfg(feature = "failpoints")] +pub use enable_failpoint::*; -impl SyncCallback { - fn new(f: impl Fn() + Send + Sync + 'static) -> SyncCallback { - SyncCallback(Arc::new(f)) +#[cfg(feature = "failpoints")] +#[macro_use] +mod enable_failpoint { + use std::collections::HashMap; + use std::env::VarError; + use std::fmt::Debug; + use std::str::FromStr; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::MutexGuard; + use std::sync::{Arc, Condvar, Mutex, RwLock, TryLockError}; + use std::time::{Duration, Instant}; + use std::{env, thread}; + + /// Registry with failpoints configuration. + type Registry = HashMap>; + + lazy_static::lazy_static! { + static ref REGISTRY_GROUP: RwLock>>> = Default::default(); + static ref REGISTRY_GLOBAL: FailPointRegistry = FailPointRegistry::default(); + static ref SCENARIO: Mutex<&'static FailPointRegistry> = Mutex::new(®ISTRY_GLOBAL); } - fn run(&self) { - let callback = &self.0; - callback(); + /// Fail point registry. Threads bound to the same registry share the same + /// failpoints configuration. + #[derive(Debug, Default, Clone)] + pub struct FailPointRegistry { + // TODO: remove rwlock or store *mut FailPoint + registry: Arc>, } -} -/// Supported tasks. -#[derive(Clone, Debug, PartialEq)] -enum Task { - /// Do nothing. - Off, - /// Return the value. - Return(Option), - /// Sleep for some milliseconds. - Sleep(u64), - /// Panic with the message. - Panic(Option), - /// Print the message. - Print(Option), - /// Sleep until other action is set. - Pause, - /// Yield the CPU. - Yield, - /// Busy waiting for some milliseconds. - Delay(u64), - /// Call callback function. - Callback(SyncCallback), -} + impl FailPointRegistry { + /// Generate a new failpoint registry. The new registry will inherit the + /// global failpoints configuration. + /// + /// Each thread should be bound to exact one registry. Threads bound to the + /// same registry share the same failpoints configuration. + pub fn new() -> Self { + FailPointRegistry { + registry: Arc::new(RwLock::new(Registry::new())), + } + } -#[derive(Debug)] -struct Action { - task: Task, - freq: f32, - count: Option, -} + /// Returns the failpoint registry to which the current thread is bound. If + /// `failpoints` feature is not enabled, the internal registry is none. + pub fn current_registry() -> Self { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group + .get(&id) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) + .clone() + }; + FailPointRegistry { registry } + } -impl PartialEq for Action { - fn eq(&self, hs: &Action) -> bool { - if self.task != hs.task || self.freq != hs.freq { - return false; + /// Register the current thread to this failpoints registry. + pub fn register_current(&self) { + let id = thread::current().id(); + REGISTRY_GROUP + .write() + .unwrap() + .insert(id, self.registry.clone()); } - if let Some(ref lhs) = self.count { - if let Some(ref rhs) = hs.count { - return lhs.load(Ordering::Relaxed) == rhs.load(Ordering::Relaxed); - } - } else if hs.count.is_none() { - return true; + + /// Deregister the current thread to this failpoints registry. + pub fn deregister_current() { + let id = thread::current().id(); + REGISTRY_GROUP.write().unwrap().remove(&id); } - false - } -} -impl Action { - fn new(task: Task, freq: f32, max_cnt: Option) -> Action { - Action { - task, - freq, - count: max_cnt.map(AtomicUsize::new), + /// Clean up registered fail points in this registry. + pub fn teardown(&self) { + let mut registry = self.registry.write().unwrap(); + cleanup(&mut registry); } } - fn from_callback(f: impl Fn() + Send + Sync + 'static) -> Action { - let task = Task::Callback(SyncCallback::new(f)); - Action { - task, - freq: 1.0, - count: None, - } + /// Test scenario with configured fail points. + #[derive(Debug)] + pub struct FailScenario<'a> { + scenario_guard: MutexGuard<'a, &'static FailPointRegistry>, } - fn get_task(&self) -> Option { - use rand::Rng; + impl<'a> FailScenario<'a> { + /// Set up the system for a fail points scenario. + /// + /// Configures global fail points specified in the `FAILPOINTS` environment variable. + /// It does not otherwise change any existing fail point configuration. + /// + /// The format of `FAILPOINTS` is `failpoint=actions;...`, where + /// `failpoint` is the name of the fail point. For more information + /// about fail point actions see the [`cfg`](fn.cfg.html) function and + /// the [`fail_point`](macro.fail_point.html) macro. + /// + /// `FAILPOINTS` may configure fail points that are not actually defined. In + /// this case the configuration has no effect. + /// + /// This function should generally be called prior to running a test with fail + /// points, and afterward paired with [`teardown`](#method.teardown). + /// + /// # Panics + /// + /// Panics if an action is not formatted correctly. + pub fn setup() -> Self { + // Cleanup first, in case of previous failed/panic'ed test scenarios. + let scenario_guard = SCENARIO.lock().unwrap_or_else(|e| e.into_inner()); + let mut registry = scenario_guard.registry.write().unwrap(); + cleanup(&mut registry); - if let Some(ref cnt) = self.count { - let c = cnt.load(Ordering::Acquire); - if c == 0 { - return None; - } - } - if self.freq < 1f32 && !rand::thread_rng().gen_bool(f64::from(self.freq)) { - return None; - } - if let Some(ref cnt) = self.count { - loop { - let c = cnt.load(Ordering::Acquire); - if c == 0 { - return None; + let failpoints = match env::var("FAILPOINTS") { + Ok(s) => s, + Err(VarError::NotPresent) => return Self { scenario_guard }, + Err(e) => panic!("invalid failpoints: {:?}", e), + }; + for mut cfg in failpoints.trim().split(';') { + cfg = cfg.trim(); + if cfg.is_empty() { + continue; } - if c == cnt.compare_and_swap(c, c - 1, Ordering::AcqRel) { - break; + let (name, order) = partition(cfg, '='); + match order { + None => panic!("invalid failpoint: {:?}", cfg), + Some(order) => { + if let Err(e) = set(&mut registry, name.to_owned(), order) { + panic!("unable to configure failpoint \"{}\": {}", name, e); + } + } } } + Self { scenario_guard } } - Some(self.task.clone()) - } -} - -fn partition(s: &str, pattern: char) -> (&str, Option<&str>) { - let mut splits = s.splitn(2, pattern); - (splits.next().unwrap(), splits.next()) -} -impl FromStr for Action { - type Err = String; - - /// Parse an action. - /// - /// `s` should be in the format `[p%][cnt*]task[(args)]`, `p%` is the frequency, - /// `cnt` is the max times the action can be triggered. - fn from_str(s: &str) -> Result { - let mut remain = s.trim(); - let mut args = None; - // in case there is '%' in args, we need to parse it first. - let (first, second) = partition(remain, '('); - if let Some(second) = second { - remain = first; - if !second.ends_with(')') { - return Err("parentheses do not match".to_owned()); - } - args = Some(&second[..second.len() - 1]); + /// Tear down the global fail points registry. + /// + /// Clears the configuration of global fail points. Any paused fail + /// points will be notified before they are deactivated. + /// + /// This function should generally be called after running a test with fail points. + /// Calling `teardown` without previously calling `setup` results in a no-op. + pub fn teardown(self) { + drop(self); } + } - let mut frequency = 1f32; - let (first, second) = partition(remain, '%'); - if let Some(second) = second { - remain = second; - match first.parse::() { - Err(e) => return Err(format!("failed to parse frequency: {}", e)), - Ok(freq) => frequency = freq / 100.0, - } + impl<'a> Drop for FailScenario<'a> { + fn drop(&mut self) { + let mut registry = self.scenario_guard.registry.write().unwrap(); + cleanup(&mut registry); } + } - let mut max_cnt = None; - let (first, second) = partition(remain, '*'); - if let Some(second) = second { - remain = second; - match first.parse() { - Err(e) => return Err(format!("failed to parse count: {}", e)), - Ok(cnt) => max_cnt = Some(cnt), - } + /// Clean all registered fail points. + fn cleanup(registry: &mut std::sync::RwLockWriteGuard) { + for p in registry.values() { + // wake up all pause failpoint. + p.set_actions("", vec![]); } + registry.clear(); + } - let parse_timeout = || match args { - None => Err("sleep require timeout".to_owned()), - Some(timeout_str) => match timeout_str.parse() { - Err(e) => Err(format!("failed to parse timeout: {}", e)), - Ok(timeout) => Ok(timeout), - }, + /// Get all registered fail points in current registry. + /// + /// If current thread is not bound to any local registry. It will try to + /// get from the global registry. + /// + /// Return a vector of `(name, actions)` pairs. + pub fn list() -> Vec<(String, String)> { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group + .get(&id) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) + .clone() }; - let task = match remain { - "off" => Task::Off, - "return" => Task::Return(args.map(str::to_owned)), - "sleep" => Task::Sleep(parse_timeout()?), - "panic" => Task::Panic(args.map(str::to_owned)), - "print" => Task::Print(args.map(str::to_owned)), - "pause" => Task::Pause, - "yield" => Task::Yield, - "delay" => Task::Delay(parse_timeout()?), - _ => return Err(format!("unrecognized command {:?}", remain)), - }; + let registry = registry.read().unwrap(); - Ok(Action::new(task, frequency, max_cnt)) + registry + .iter() + .map(|(name, fp)| (name.to_string(), fp.actions_str.read().unwrap().clone())) + .collect() } -} -#[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] -#[derive(Debug, Default)] -struct FailPoint { - pause: Mutex, - pause_notifier: Condvar, - actions: RwLock>, - actions_str: RwLock, -} - -#[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] -impl FailPoint { - fn new() -> FailPoint { - FailPoint::default() - } + #[doc(hidden)] + pub fn eval) -> R>(name: &str, f: F) -> Option { + let p = { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group + .get(&id) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) + .clone() + }; - fn set_actions(&self, actions_str: &str, actions: Vec) { - loop { - // TODO: maybe busy waiting here. - match self.actions.try_write() { - Err(TryLockError::WouldBlock) => {} - Ok(mut guard) => { - *guard = actions; - *self.actions_str.write().unwrap() = actions_str.to_string(); - return; - } - Err(e) => panic!("unexpected poison: {:?}", e), - } - let mut guard = self.pause.lock().unwrap(); - *guard = false; - self.pause_notifier.notify_all(); - } - } + let registry = registry.read().unwrap(); - #[cfg_attr(feature = "cargo-clippy", allow(clippy::option_option))] - fn eval(&self, name: &str) -> Option> { - let task = { - let actions = self.actions.read().unwrap(); - match actions.iter().filter_map(Action::get_task).next() { - Some(Task::Pause) => { - let mut guard = self.pause.lock().unwrap(); - *guard = true; - loop { - guard = self.pause_notifier.wait(guard).unwrap(); - if !*guard { - break; - } - } - return None; - } - Some(t) => t, + match registry.get(name) { None => return None, + Some(p) => p.clone(), } }; - - match task { - Task::Off => {} - Task::Return(s) => return Some(s), - Task::Sleep(t) => thread::sleep(Duration::from_millis(t)), - Task::Panic(msg) => match msg { - Some(ref msg) => panic!("{}", msg), - None => panic!("failpoint {} panic", name), - }, - Task::Print(msg) => match msg { - Some(ref msg) => log::info!("{}", msg), - None => log::info!("failpoint {} executed.", name), - }, - Task::Pause => unreachable!(), - Task::Yield => thread::yield_now(), - Task::Delay(t) => { - let timer = Instant::now(); - let timeout = Duration::from_millis(t); - while timer.elapsed() < timeout {} - } - Task::Callback(f) => { - f.run(); - } - } - None + p.eval(name).map(f) } -} -/// Registry with failpoints configuration. -type Registry = HashMap>; - -/// Fail point registry. Threads bound to the same registry share the same -/// failpoints configuration. -#[derive(Debug, Default, Clone)] -pub struct FailPointRegistry { - // TODO: remove rwlock or store *mut FailPoint - registry: Option>>, -} - -impl FailPointRegistry { - /// Generate a new failpoint registry. The new registry will inherit the - /// global failpoints configuration. + /// Configure the actions for a fail point in current registry at runtime. /// - /// Each thread should be bound to exact one registry. Threads bound to the - /// same registry share the same failpoints configuration. - pub fn new() -> Self { - FailPointRegistry { - registry: Some(Arc::new(RwLock::new(Registry::new()))), - } - } - - /// Returns the failpoint registry to which the current thread is bound. If - /// `failpoints` feature is not enabled, the internal registry is none. - pub fn current_registry() -> Self { + /// Each fail point can be configured with a series of actions, specified by the + /// `actions` argument. The format of `actions` is `action[->action...]`. When + /// multiple actions are specified, an action will be checked only when its + /// former action is not triggered. + /// + /// The format of a single action is `[p%][cnt*]task[(arg)]`. `p%` is the + /// expected probability that the action is triggered, and `cnt*` is the max + /// times the action can be triggered. The supported values of `task` are: + /// + /// - `off`, the fail point will do nothing. + /// - `return(arg)`, return early when the fail point is triggered. `arg` is passed to `$e` ( + /// defined via the `fail_point!` macro) as a string. + /// - `sleep(milliseconds)`, sleep for the specified time. + /// - `panic(msg)`, panic with the message. + /// - `print(msg)`, log the message, using the `log` crate, at the `info` level. + /// - `pause`, sleep until other action is set to the fail point. + /// - `yield`, yield the CPU. + /// - `delay(milliseconds)`, busy waiting for the specified time. + /// + /// For example, `20%3*print(still alive!)->panic` means the fail point has 20% chance to print a + /// message "still alive!" and 80% chance to panic. And the message will be printed at most 3 + /// times. + /// + /// The `FAILPOINTS` environment variable accepts this same syntax for its fail + /// point actions. + /// + /// A call to `cfg` with a particular fail point name overwrites any existing actions for + /// that fail point, including those set via the `FAILPOINTS` environment variable. + pub fn cfg>(name: S, actions: &str) -> Result<(), String> { if cfg!(feature = "failpoints") { let registry = { let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); group .get(&id) - .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) .clone() }; - FailPointRegistry { - registry: Some(registry), - } + + let mut registry = registry.write().unwrap(); + + set(&mut registry, name.into(), actions) } else { - FailPointRegistry { registry: None } + Err("failpoints is not enabled".to_owned()) } } - /// Register the current thread to this failpoints registry. - pub fn register_current(&self) { - if cfg!(feature = "failpoints") { + /// Configure the actions for a fail point in current registry at runtime. + /// + /// Each fail point can be configured by a callback. Process will call this callback function + /// when it meet this fail-point. + pub fn cfg_callback(name: S, f: F) -> Result<(), String> + where + S: Into, + F: Fn() + Send + Sync + 'static, + { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - REGISTRY_GROUP - .write() - .unwrap() - .insert(id, self.registry.clone().unwrap()); - } + group + .get(&id) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) + .clone() + }; + + let mut registry = registry.write().unwrap(); + + let p = registry + .entry(name.into()) + .or_insert_with(|| Arc::new(FailPoint::new())); + let action = Action::from_callback(f); + let actions = vec![action]; + p.set_actions("callback", actions); + Ok(()) } - /// Deregister the current thread to this failpoints registry. - pub fn deregister_current() { - if cfg!(feature = "failpoints") { + /// Remove a fail point. + /// + /// If the local registry doesn't exist, it will try to delete the corresponding + /// action in the global registry. + pub fn remove>(name: S) { + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); let id = thread::current().id(); - REGISTRY_GROUP.write().unwrap().remove(&id); - } - } + group + .get(&id) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) + .clone() + }; - /// Clean up registered fail points in this registry. - pub fn teardown(&self) { - if cfg!(feature = "failpoints") { - let mut registry = self.registry.as_ref().unwrap().write().unwrap(); - cleanup(&mut registry); + let mut registry = registry.write().unwrap(); + + if let Some(p) = registry.remove(name.as_ref()) { + // wake up all pause failpoint. + p.set_actions("", vec![]); } } -} - -lazy_static::lazy_static! { - static ref REGISTRY_GROUP: RwLock>>> = Default::default(); - static ref REGISTRY_GLOBAL: FailPointRegistry = FailPointRegistry { registry: Some(Default::default()) }; - static ref SCENARIO: Mutex<&'static FailPointRegistry> = Mutex::new(®ISTRY_GLOBAL); -} -/// Test scenario with configured fail points. -#[derive(Debug)] -pub struct FailScenario<'a> { - scenario_guard: MutexGuard<'a, &'static FailPointRegistry>, -} + fn set( + registry: &mut HashMap>, + name: String, + actions: &str, + ) -> Result<(), String> { + let actions_str = actions; + // `actions` are in the format of `failpoint[->failpoint...]`. + let actions = actions + .split("->") + .map(Action::from_str) + .collect::>()?; + // Please note that we can't figure out whether there is a failpoint named `name`, + // so we may insert a failpoint that doesn't exist at all. + let p = registry + .entry(name) + .or_insert_with(|| Arc::new(FailPoint::new())); + p.set_actions(actions_str, actions); + Ok(()) + } -impl<'a> FailScenario<'a> { - /// Set up the system for a fail points scenario. + /// Define a fail point (requires `failpoints` feature). /// - /// Configures global fail points specified in the `FAILPOINTS` environment variable. - /// It does not otherwise change any existing fail point configuration. + /// The `fail_point!` macro has three forms, and they all take a name as the + /// first argument. The simplest form takes only a name and is suitable for + /// executing most fail point behavior, including panicking, but not for early + /// return or conditional execution based on a local flag. /// - /// The format of `FAILPOINTS` is `failpoint=actions;...`, where - /// `failpoint` is the name of the fail point. For more information - /// about fail point actions see the [`cfg`](fn.cfg.html) function and - /// the [`fail_point`](macro.fail_point.html) macro. + /// The three forms of fail points look as follows. /// - /// `FAILPOINTS` may configure fail points that are not actually defined. In - /// this case the configuration has no effect. + /// 1. A basic fail point: /// - /// This function should generally be called prior to running a test with fail - /// points, and afterward paired with [`teardown`](#method.teardown). + /// ```rust + /// # #[macro_use] extern crate fail; + /// fn function_return_unit() { + /// fail_point!("fail-point-1"); + /// } + /// ``` /// - /// # Panics + /// This form of fail point can be configured to panic, print, sleep, pause, etc., but + /// not to return from the function early. /// - /// Panics if an action is not formatted correctly. - pub fn setup() -> Self { - // Cleanup first, in case of previous failed/panic'ed test scenarios. - let scenario_guard = SCENARIO.lock().unwrap_or_else(|e| e.into_inner()); - let mut registry = scenario_guard.registry.as_ref().unwrap().write().unwrap(); - cleanup(&mut registry); - - let failpoints = match env::var("FAILPOINTS") { - Ok(s) => s, - Err(VarError::NotPresent) => return Self { scenario_guard }, - Err(e) => panic!("invalid failpoints: {:?}", e), - }; - for mut cfg in failpoints.trim().split(';') { - cfg = cfg.trim(); - if cfg.is_empty() { - continue; + /// 2. A fail point that may return early: + /// + /// ```rust + /// # #[macro_use] extern crate fail; + /// fn function_return_value() -> u64 { + /// fail_point!("fail-point-2", |r| r.map_or(2, |e| e.parse().unwrap())); + /// 0 + /// } + /// ``` + /// + /// This form of fail point can additionally be configured to return early from + /// the enclosing function. It accepts a closure, which itself accepts an + /// `Option`, and is expected to transform that argument into the early + /// return value. The argument string is sourced from the fail point + /// configuration string. For example configuring this "fail-point-2" as + /// "return(100)" will execute the fail point closure, passing it a `Some` value + /// containing a `String` equal to "100"; the closure then parses it into the + /// return value. + /// + /// 3. A fail point with conditional execution: + /// + /// ```rust + /// # #[macro_use] extern crate fail; + /// fn function_conditional(enable: bool) { + /// fail_point!("fail-point-3", enable, |_| {}); + /// } + /// ``` + /// + /// In this final form, the second argument is a local boolean expression that + /// must evaluate to `true` before the fail point is evaluated. The third + /// argument is again an early-return closure. + /// + /// The three macro arguments (or "designators") are called `$name`, `$cond`, + /// and `$e`. `$name` must be `&str`, `$cond` must be a boolean expression, + /// and`$e` must be a function or closure that accepts an `Option` and + /// returns the same type as the enclosing function. + /// + /// For more examples see the [crate documentation](index.html). For more + /// information about controlling fail points see the [`cfg`](fn.cfg.html) + /// function. + #[macro_export] + macro_rules! fail_point { + ($name:expr) => {{ + $crate::eval($name, |_| { + panic!("Return is not supported for the fail point \"{}\"", $name); + }); + }}; + ($name:expr, $e:expr) => {{ + if let Some(res) = $crate::eval($name, $e) { + return res; } - let (name, order) = partition(cfg, '='); - match order { - None => panic!("invalid failpoint: {:?}", cfg), - Some(order) => { - if let Err(e) = set(&mut registry, name.to_owned(), order) { - panic!("unable to configure failpoint \"{}\": {}", name, e); - } - } + }}; + ($name:expr, $cond:expr, $e:expr) => {{ + if $cond { + fail_point!($name, $e); } - } - Self { scenario_guard } + }}; } - /// Tear down the global fail points registry. - /// - /// Clears the configuration of global fail points. Any paused fail - /// points will be notified before they are deactivated. - /// - /// This function should generally be called after running a test with fail points. - /// Calling `teardown` without previously calling `setup` results in a no-op. - pub fn teardown(self) { - drop(self); + #[derive(Clone)] + pub(crate) struct SyncCallback(Arc); + + impl Debug for SyncCallback { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SyncCallback()") + } } -} -impl<'a> Drop for FailScenario<'a> { - fn drop(&mut self) { - let mut registry = self - .scenario_guard - .registry - .as_ref() - .unwrap() - .write() - .unwrap(); - cleanup(&mut registry); + impl PartialEq for SyncCallback { + fn eq(&self, _: &Self) -> bool { + unimplemented!() + } } -} -/// Clean all registered fail points. -fn cleanup(registry: &mut std::sync::RwLockWriteGuard) { - for p in registry.values() { - // wake up all pause failpoint. - p.set_actions("", vec![]); + impl SyncCallback { + fn new(f: impl Fn() + Send + Sync + 'static) -> SyncCallback { + SyncCallback(Arc::new(f)) + } + + fn run(&self) { + let callback = &self.0; + callback(); + } } - registry.clear(); -} -/// Returns whether code generation for failpoints is enabled. -/// -/// This function allows consumers to check (at runtime) whether the library -/// was compiled with the (buildtime) `failpoints` feature, which enables -/// code generation for failpoints. -pub const fn has_failpoints() -> bool { - cfg!(feature = "failpoints") -} + /// Supported tasks. + #[derive(Clone, Debug, PartialEq)] + pub(crate) enum Task { + /// Do nothing. + Off, + /// Return the value. + Return(Option), + /// Sleep for some milliseconds. + Sleep(u64), + /// Panic with the message. + Panic(Option), + /// Print the message. + Print(Option), + /// Sleep until other action is set. + Pause, + /// Yield the CPU. + Yield, + /// Busy waiting for some milliseconds. + Delay(u64), + /// Call callback function. + Callback(SyncCallback), + } -/// Get all registered fail points in current registry. -/// -/// If current thread is not bound to any local registry. It will try to -/// get from the global registry. -/// -/// Return a vector of `(name, actions)` pairs. -pub fn list() -> Vec<(String, String)> { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) - .clone() - }; - - let registry = registry.read().unwrap(); - - registry - .iter() - .map(|(name, fp)| (name.to_string(), fp.actions_str.read().unwrap().clone())) - .collect() -} + #[derive(Debug)] + pub(crate) struct Action { + task: Task, + freq: f32, + count: Option, + } -#[doc(hidden)] -pub fn eval) -> R>(name: &str, f: F) -> Option { - let p = { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) - .clone() - }; + impl PartialEq for Action { + fn eq(&self, hs: &Action) -> bool { + if self.task != hs.task || self.freq != hs.freq { + return false; + } + if let Some(ref lhs) = self.count { + if let Some(ref rhs) = hs.count { + return lhs.load(Ordering::Relaxed) == rhs.load(Ordering::Relaxed); + } + } else if hs.count.is_none() { + return true; + } + false + } + } - let registry = registry.read().unwrap(); + impl Action { + pub(crate) fn new(task: Task, freq: f32, max_cnt: Option) -> Action { + Action { + task, + freq, + count: max_cnt.map(AtomicUsize::new), + } + } - match registry.get(name) { - None => return None, - Some(p) => p.clone(), + pub(crate) fn from_callback(f: impl Fn() + Send + Sync + 'static) -> Action { + let task = Task::Callback(SyncCallback::new(f)); + Action { + task, + freq: 1.0, + count: None, + } } - }; - p.eval(name).map(f) -} -/// Configure the actions for a fail point in current registry at runtime. -/// -/// Each fail point can be configured with a series of actions, specified by the -/// `actions` argument. The format of `actions` is `action[->action...]`. When -/// multiple actions are specified, an action will be checked only when its -/// former action is not triggered. -/// -/// The format of a single action is `[p%][cnt*]task[(arg)]`. `p%` is the -/// expected probability that the action is triggered, and `cnt*` is the max -/// times the action can be triggered. The supported values of `task` are: -/// -/// - `off`, the fail point will do nothing. -/// - `return(arg)`, return early when the fail point is triggered. `arg` is passed to `$e` ( -/// defined via the `fail_point!` macro) as a string. -/// - `sleep(milliseconds)`, sleep for the specified time. -/// - `panic(msg)`, panic with the message. -/// - `print(msg)`, log the message, using the `log` crate, at the `info` level. -/// - `pause`, sleep until other action is set to the fail point. -/// - `yield`, yield the CPU. -/// - `delay(milliseconds)`, busy waiting for the specified time. -/// -/// For example, `20%3*print(still alive!)->panic` means the fail point has 20% chance to print a -/// message "still alive!" and 80% chance to panic. And the message will be printed at most 3 -/// times. -/// -/// The `FAILPOINTS` environment variable accepts this same syntax for its fail -/// point actions. -/// -/// A call to `cfg` with a particular fail point name overwrites any existing actions for -/// that fail point, including those set via the `FAILPOINTS` environment variable. -pub fn cfg>(name: S, actions: &str) -> Result<(), String> { - if cfg!(feature = "failpoints") { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) - .clone() - }; + pub(crate) fn get_task(&self) -> Option { + use rand::Rng; - let mut registry = registry.write().unwrap(); + if let Some(ref cnt) = self.count { + let c = cnt.load(Ordering::Acquire); + if c == 0 { + return None; + } + } + if self.freq < 1f32 && !rand::thread_rng().gen_bool(f64::from(self.freq)) { + return None; + } + if let Some(ref cnt) = self.count { + loop { + let c = cnt.load(Ordering::Acquire); + if c == 0 { + return None; + } + if c == cnt.compare_and_swap(c, c - 1, Ordering::AcqRel) { + break; + } + } + } + Some(self.task.clone()) + } + } - set(&mut registry, name.into(), actions) - } else { - Err("failpoints is not enabled".to_owned()) + fn partition(s: &str, pattern: char) -> (&str, Option<&str>) { + let mut splits = s.splitn(2, pattern); + (splits.next().unwrap(), splits.next()) } -} -/// Configure the actions for a fail point in current registry at runtime. -/// -/// Each fail point can be configured by a callback. Process will call this callback function -/// when it meet this fail-point. -pub fn cfg_callback(name: S, f: F) -> Result<(), String> -where - S: Into, - F: Fn() + Send + Sync + 'static, -{ - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) - .clone() - }; - - let mut registry = registry.write().unwrap(); - - let p = registry - .entry(name.into()) - .or_insert_with(|| Arc::new(FailPoint::new())); - let action = Action::from_callback(f); - let actions = vec![action]; - p.set_actions("callback", actions); - Ok(()) -} + impl FromStr for Action { + type Err = String; + + /// Parse an action. + /// + /// `s` should be in the format `[p%][cnt*]task[(args)]`, `p%` is the frequency, + /// `cnt` is the max times the action can be triggered. + fn from_str(s: &str) -> Result { + let mut remain = s.trim(); + let mut args = None; + // in case there is '%' in args, we need to parse it first. + let (first, second) = partition(remain, '('); + if let Some(second) = second { + remain = first; + if !second.ends_with(')') { + return Err("parentheses do not match".to_owned()); + } + args = Some(&second[..second.len() - 1]); + } -/// Remove a fail point. -/// -/// If the local registry doesn't exist, it will try to delete the corresponding -/// action in the global registry. -pub fn remove>(name: S) { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or_else(|| REGISTRY_GLOBAL.registry.as_ref().unwrap()) - .clone() - }; - - let mut registry = registry.write().unwrap(); - - if let Some(p) = registry.remove(name.as_ref()) { - // wake up all pause failpoint. - p.set_actions("", vec![]); + let mut frequency = 1f32; + let (first, second) = partition(remain, '%'); + if let Some(second) = second { + remain = second; + match first.parse::() { + Err(e) => return Err(format!("failed to parse frequency: {}", e)), + Ok(freq) => frequency = freq / 100.0, + } + } + + let mut max_cnt = None; + let (first, second) = partition(remain, '*'); + if let Some(second) = second { + remain = second; + match first.parse() { + Err(e) => return Err(format!("failed to parse count: {}", e)), + Ok(cnt) => max_cnt = Some(cnt), + } + } + + let parse_timeout = || match args { + None => Err("sleep require timeout".to_owned()), + Some(timeout_str) => match timeout_str.parse() { + Err(e) => Err(format!("failed to parse timeout: {}", e)), + Ok(timeout) => Ok(timeout), + }, + }; + + let task = match remain { + "off" => Task::Off, + "return" => Task::Return(args.map(str::to_owned)), + "sleep" => Task::Sleep(parse_timeout()?), + "panic" => Task::Panic(args.map(str::to_owned)), + "print" => Task::Print(args.map(str::to_owned)), + "pause" => Task::Pause, + "yield" => Task::Yield, + "delay" => Task::Delay(parse_timeout()?), + _ => return Err(format!("unrecognized command {:?}", remain)), + }; + + Ok(Action::new(task, frequency, max_cnt)) + } } -} -fn set( - registry: &mut HashMap>, - name: String, - actions: &str, -) -> Result<(), String> { - let actions_str = actions; - // `actions` are in the format of `failpoint[->failpoint...]`. - let actions = actions - .split("->") - .map(Action::from_str) - .collect::>()?; - // Please note that we can't figure out whether there is a failpoint named `name`, - // so we may insert a failpoint that doesn't exist at all. - let p = registry - .entry(name) - .or_insert_with(|| Arc::new(FailPoint::new())); - p.set_actions(actions_str, actions); - Ok(()) -} + #[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] + #[derive(Debug, Default)] + pub(crate) struct FailPoint { + pause: Mutex, + pause_notifier: Condvar, + actions: RwLock>, + actions_str: RwLock, + } -/// Define a fail point (requires `failpoints` feature). -/// -/// The `fail_point!` macro has three forms, and they all take a name as the -/// first argument. The simplest form takes only a name and is suitable for -/// executing most fail point behavior, including panicking, but not for early -/// return or conditional execution based on a local flag. -/// -/// The three forms of fail points look as follows. -/// -/// 1. A basic fail point: -/// -/// ```rust -/// # #[macro_use] extern crate fail; -/// fn function_return_unit() { -/// fail_point!("fail-point-1"); -/// } -/// ``` -/// -/// This form of fail point can be configured to panic, print, sleep, pause, etc., but -/// not to return from the function early. -/// -/// 2. A fail point that may return early: -/// -/// ```rust -/// # #[macro_use] extern crate fail; -/// fn function_return_value() -> u64 { -/// fail_point!("fail-point-2", |r| r.map_or(2, |e| e.parse().unwrap())); -/// 0 -/// } -/// ``` -/// -/// This form of fail point can additionally be configured to return early from -/// the enclosing function. It accepts a closure, which itself accepts an -/// `Option`, and is expected to transform that argument into the early -/// return value. The argument string is sourced from the fail point -/// configuration string. For example configuring this "fail-point-2" as -/// "return(100)" will execute the fail point closure, passing it a `Some` value -/// containing a `String` equal to "100"; the closure then parses it into the -/// return value. -/// -/// 3. A fail point with conditional execution: -/// -/// ```rust -/// # #[macro_use] extern crate fail; -/// fn function_conditional(enable: bool) { -/// fail_point!("fail-point-3", enable, |_| {}); -/// } -/// ``` -/// -/// In this final form, the second argument is a local boolean expression that -/// must evaluate to `true` before the fail point is evaluated. The third -/// argument is again an early-return closure. -/// -/// The three macro arguments (or "designators") are called `$name`, `$cond`, -/// and `$e`. `$name` must be `&str`, `$cond` must be a boolean expression, -/// and`$e` must be a function or closure that accepts an `Option` and -/// returns the same type as the enclosing function. -/// -/// For more examples see the [crate documentation](index.html). For more -/// information about controlling fail points see the [`cfg`](fn.cfg.html) -/// function. -#[macro_export] -#[cfg(feature = "failpoints")] -macro_rules! fail_point { - ($name:expr) => {{ - $crate::eval($name, |_| { - panic!("Return is not supported for the fail point \"{}\"", $name); - }); - }}; - ($name:expr, $e:expr) => {{ - if let Some(res) = $crate::eval($name, $e) { - return res; + #[cfg_attr(feature = "cargo-clippy", allow(clippy::mutex_atomic))] + impl FailPoint { + pub(crate) fn new() -> FailPoint { + FailPoint::default() } - }}; - ($name:expr, $cond:expr, $e:expr) => {{ - if $cond { - fail_point!($name, $e); + + pub(crate) fn set_actions(&self, actions_str: &str, actions: Vec) { + loop { + // TODO: maybe busy waiting here. + match self.actions.try_write() { + Err(TryLockError::WouldBlock) => {} + Ok(mut guard) => { + *guard = actions; + *self.actions_str.write().unwrap() = actions_str.to_string(); + return; + } + Err(e) => panic!("unexpected poison: {:?}", e), + } + let mut guard = self.pause.lock().unwrap(); + *guard = false; + self.pause_notifier.notify_all(); + } + } + + #[cfg_attr(feature = "cargo-clippy", allow(clippy::option_option))] + pub(crate) fn eval(&self, name: &str) -> Option> { + let task = { + let actions = self.actions.read().unwrap(); + match actions.iter().filter_map(Action::get_task).next() { + Some(Task::Pause) => { + let mut guard = self.pause.lock().unwrap(); + *guard = true; + loop { + guard = self.pause_notifier.wait(guard).unwrap(); + if !*guard { + break; + } + } + return None; + } + Some(t) => t, + None => return None, + } + }; + + match task { + Task::Off => {} + Task::Return(s) => return Some(s), + Task::Sleep(t) => thread::sleep(Duration::from_millis(t)), + Task::Panic(msg) => match msg { + Some(ref msg) => panic!("{}", msg), + None => panic!("failpoint {} panic", name), + }, + Task::Print(msg) => match msg { + Some(ref msg) => log::info!("{}", msg), + None => log::info!("failpoint {} executed.", name), + }, + Task::Pause => unreachable!(), + Task::Yield => thread::yield_now(), + Task::Delay(t) => { + let timer = Instant::now(); + let timeout = Duration::from_millis(t); + while timer.elapsed() < timeout {} + } + Task::Callback(f) => { + f.run(); + } + } + None } - }}; + } } -/// Define a fail point (disabled, see `failpoints` feature). -#[macro_export] #[cfg(not(feature = "failpoints"))] -macro_rules! fail_point { - ($name:expr, $e:expr) => {{}}; - ($name:expr) => {{}}; - ($name:expr, $cond:expr, $e:expr) => {{}}; +mod disable_failpoint { + #[derive(Debug, Default, Clone)] + /// Define a fail point registry (disabled, see `failpoints` feature). + pub struct FailPointRegistry; + + impl FailPointRegistry { + /// Generate a new failpoint registry (disabled, see `failpoints` feature). + pub fn new() -> Self { + FailPointRegistry::default() + } + + /// Returns the failpoint registry to which the current thread is bound + /// (disabled, see `failpoints` feature). + pub fn current_registry() -> Self { + FailPointRegistry::default() + } + + /// Register the current thread (disabled, see `failpoints` feature). + pub fn register_current(&self) {} + + /// Deregister the current thread (disabled, see `failpoints` feature). + pub fn deregister_current() {} + + /// Clean up fail points in this registry (disabled, see `failpoints` feature). + pub fn teardown(&self) {} + } + + /// Define a fail point (disabled, see `failpoints` feature). + #[macro_export] + macro_rules! fail_point { + ($name:expr, $e:expr) => {{}}; + ($name:expr) => {{}}; + ($name:expr, $cond:expr, $e:expr) => {{}}; + } +} + +/// Returns whether code generation for failpoints is enabled. +/// +/// This function allows consumers to check (at runtime) whether the library +/// was compiled with the (buildtime) `failpoints` feature, which enables +/// code generation for failpoints. +pub const fn has_failpoints() -> bool { + cfg!(feature = "failpoints") } #[cfg(test)] +#[cfg(feature = "failpoints")] mod tests { + use super::enable_failpoint::*; use super::*; - + use std::env; use std::sync::*; + use std::thread; + use std::time::{Duration, Instant}; #[test] fn test_has_failpoints() { @@ -1153,7 +1175,6 @@ mod tests { // This case should be tested as integration case, but when calling `teardown` other cases // like `test_pause` maybe also affected, so it's better keep it here. #[test] - #[cfg_attr(not(feature = "failpoints"), ignore)] fn test_setup_and_teardown() { let f1 = || { fail_point!("setup_and_teardown1", |_| 1); diff --git a/tests/tests.rs b/tests/tests.rs index 98d682b..a8c636d 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -2,13 +2,13 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::*; +use std::thread; use std::time::*; -use std::*; use fail::fail_point; #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_pause() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -42,7 +42,7 @@ fn test_pause() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_off() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -58,7 +58,7 @@ fn test_off() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_return() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -78,7 +78,7 @@ fn test_return() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_sleep() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -98,7 +98,7 @@ fn test_sleep() { #[test] #[should_panic] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_panic() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -111,7 +111,7 @@ fn test_panic() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_print() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -148,7 +148,7 @@ fn test_print() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_yield() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -161,7 +161,7 @@ fn test_yield() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_callback() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -185,7 +185,7 @@ fn test_callback() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_delay() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -198,7 +198,7 @@ fn test_delay() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_freq_and_count() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -222,7 +222,7 @@ fn test_freq_and_count() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_condition() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -240,7 +240,7 @@ fn test_condition() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_list() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); @@ -253,7 +253,7 @@ fn test_list() { } #[test] -#[cfg_attr(not(feature = "failpoints"), ignore)] +#[cfg(feature = "failpoints")] fn test_multiple_threads_cleanup() { let local_registry = fail::FailPointRegistry::new(); local_registry.register_current(); From 15efaf16831220b7090b311b1be67b67aaa73f61 Mon Sep 17 00:00:00 2001 From: Xintao Date: Wed, 5 Aug 2020 14:49:23 +0800 Subject: [PATCH 15/15] add api for disabled failpoints Signed-off-by: Xintao --- src/lib.rs | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d8cf185..64d6b97 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -505,22 +505,18 @@ mod enable_failpoint { /// A call to `cfg` with a particular fail point name overwrites any existing actions for /// that fail point, including those set via the `FAILPOINTS` environment variable. pub fn cfg>(name: S, actions: &str) -> Result<(), String> { - if cfg!(feature = "failpoints") { - let registry = { - let group = REGISTRY_GROUP.read().unwrap(); - let id = thread::current().id(); - group - .get(&id) - .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) - .clone() - }; + let registry = { + let group = REGISTRY_GROUP.read().unwrap(); + let id = thread::current().id(); + group + .get(&id) + .unwrap_or_else(|| ®ISTRY_GLOBAL.registry) + .clone() + }; - let mut registry = registry.write().unwrap(); + let mut registry = registry.write().unwrap(); - set(&mut registry, name.into(), actions) - } else { - Err("failpoints is not enabled".to_owned()) - } + set(&mut registry, name.into(), actions) } /// Configure the actions for a fail point in current registry at runtime. @@ -970,6 +966,21 @@ mod disable_failpoint { pub fn teardown(&self) {} } + /// Configure the actions for a fail point in current registry at runtime. + /// (disabled, see `failpoints` feature). + pub fn cfg>(_name: S, _actions: &str) -> Result<(), String> { + Err("failpoints is not enabled".to_owned()) + } + + /// Remove a fail point (disabled, see `failpoints` feature). + pub fn remove>(_name: S) {} + + /// Get all registered fail points in current registry. + /// (disabled, see `failpoints` feature). + pub fn list() -> Vec<(String, String)> { + vec![] + } + /// Define a fail point (disabled, see `failpoints` feature). #[macro_export] macro_rules! fail_point {