From 8110c76371ac343bd2fab081ba543db5271e8951 Mon Sep 17 00:00:00 2001 From: Joerg Herbel Date: Tue, 19 Sep 2023 17:14:01 +0200 Subject: [PATCH] Rust scheduler: supervised child process execution CMK-14556 --- v2/rust/Cargo.lock | 121 ++++++++++++++++++++++++ v2/rust/Cargo.toml | 1 + v2/rust/src/child_process_supervisor.rs | 93 ++++++++++++++++++ v2/rust/src/main.rs | 2 + v2/rust/src/timeout.rs | 34 +++++++ 5 files changed, 251 insertions(+) create mode 100644 v2/rust/src/child_process_supervisor.rs create mode 100644 v2/rust/src/timeout.rs diff --git a/v2/rust/Cargo.lock b/v2/rust/Cargo.lock index 8b8512f8..1fbd37bd 100644 --- a/v2/rust/Cargo.lock +++ b/v2/rust/Cargo.lock @@ -229,6 +229,39 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "ctrlc" version = "3.4.1" @@ -239,6 +272,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "errno" version = "0.3.3" @@ -402,6 +441,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -422,6 +470,15 @@ dependencies = [ "libc", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.49.0" @@ -483,6 +540,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -535,6 +612,7 @@ dependencies = [ "quick-xml", "serde", "serde_json", + "sysinfo", ] [[package]] @@ -576,6 +654,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.188" @@ -624,6 +708,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.29.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "tempfile" version = "3.8.0" @@ -723,6 +822,28 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.48.0" diff --git a/v2/rust/Cargo.toml b/v2/rust/Cargo.toml index 45b3eb20..72e72c83 100644 --- a/v2/rust/Cargo.toml +++ b/v2/rust/Cargo.toml @@ -16,3 +16,4 @@ log = "*" quick-xml = { version = "0.30.0", features = ["serialize"] } serde = { version = "1.0.188", features = ["derive"] } serde_json = "*" +sysinfo = "*" diff --git a/v2/rust/src/child_process_supervisor.rs b/v2/rust/src/child_process_supervisor.rs new file mode 100644 index 00000000..ac6d4203 --- /dev/null +++ b/v2/rust/src/child_process_supervisor.rs @@ -0,0 +1,93 @@ +use super::termination::TerminationFlag; +use super::timeout::Timeout; +use anyhow::{bail, Context, Result}; +use log::{error, warn}; +use std::collections::{HashMap, HashSet}; +use std::process::{Child, Command, ExitStatus}; +use std::thread::sleep; +use std::time::Duration; +use sysinfo::{Pid, PidExt, Process, ProcessExt, System, SystemExt}; + +pub struct ChildProcessSupervisor<'a> { + pub command: Command, + pub timeout: u64, + pub termination_flag: &'a TerminationFlag, +} + +impl ChildProcessSupervisor<'_> { + pub fn run(mut self) -> Result { + let mut child = self.command.spawn().context("Failed to spawn subprocess")?; + let timeout = Timeout::start(self.timeout); + + loop { + if let Some(exit_status) = child + .try_wait() + .context(format!( + "Failed to query exit status of process {}, killing", + child.id() + )) + .map_err(|err| { + kill_child_process_tree(&mut child); + err + })? + { + return Ok(ChildProcessOutcome::Exited(exit_status)); + } + + if timeout.expired() { + error!("Process timed out"); + kill_child_process_tree(&mut child); + return Ok(ChildProcessOutcome::TimedOut); + } + + if self.termination_flag.should_terminate() { + warn!("Terminated"); + kill_child_process_tree(&mut child); + bail!("Terminated") + } + sleep(Duration::from_millis(250)) + } + } +} + +pub enum ChildProcessOutcome { + Exited(ExitStatus), + TimedOut, +} + +fn kill_child_process_tree(child: &mut Child) { + let mut system = System::new_all(); + system.refresh_processes(); + let _ = child.kill(); + kill_all_children(&Pid::from_u32(child.id()), system.processes()); +} + +fn kill_all_children<'a>(top_pid: &'a Pid, processes: &'a HashMap) { + let mut pids_in_tree = HashSet::from([top_pid]); + + loop { + let current_tree_size = pids_in_tree.len(); + add_and_kill_direct_children(&mut pids_in_tree, processes); + if pids_in_tree.len() == current_tree_size { + break; + } + } +} + +fn add_and_kill_direct_children<'a>( + pids_in_tree: &mut HashSet<&'a Pid>, + processes: &'a HashMap, +) { + for (pid, parent_pid, process) in processes.iter().filter_map(|(pid, process)| { + process + .parent() + .map(|parent_pid| (pid, parent_pid, process)) + }) { + { + if pids_in_tree.contains(&parent_pid) { + pids_in_tree.insert(pid); + process.kill(); + } + } + } +} diff --git a/v2/rust/src/main.rs b/v2/rust/src/main.rs index 99a85518..fd0636b8 100644 --- a/v2/rust/src/main.rs +++ b/v2/rust/src/main.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] pub mod attempt; +mod child_process_supervisor; mod cli; mod config; mod environment; @@ -8,6 +9,7 @@ pub mod parse_xml; mod results; mod setup; mod termination; +mod timeout; use anyhow::{Context, Result}; use clap::Parser; diff --git a/v2/rust/src/timeout.rs b/v2/rust/src/timeout.rs new file mode 100644 index 00000000..87834c9c --- /dev/null +++ b/v2/rust/src/timeout.rs @@ -0,0 +1,34 @@ +use std::time::Instant; + +pub struct Timeout { + start_time: Instant, + timeout: u64, +} + +impl Timeout { + pub fn start(timeout: u64) -> Self { + Self { + start_time: Instant::now(), + timeout, + } + } + + pub fn expired(&self) -> bool { + self.start_time.elapsed().as_secs() >= self.timeout + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_timout_expired() { + assert!(Timeout::start(0).expired()) + } + + #[test] + fn test_timout_not_expired() { + assert!(!Timeout::start(10).expired()) + } +}