diff --git a/v2/robotmk/src/bin/scheduler/build.rs b/v2/robotmk/src/bin/scheduler/build.rs new file mode 100644 index 00000000..9931657c --- /dev/null +++ b/v2/robotmk/src/bin/scheduler/build.rs @@ -0,0 +1,192 @@ +use super::internal_config::{GlobalConfig, Suite}; +use super::logging::log_and_return_error; +use robotmk::child_process_supervisor::{ChildProcessOutcome, ChildProcessSupervisor, StdioPaths}; +use robotmk::command_spec::CommandSpec; +use robotmk::environment::{apply_current_settings, Environment, RCCEnvironment}; +use robotmk::results::{BuildOutcome, BuildStates, EnvironmentBuildStage}; + +use robotmk::lock::Locker; +use robotmk::section::WriteSection; + +use anyhow::{bail, Result}; +use camino::{Utf8Path, Utf8PathBuf}; +use chrono::{DateTime, Utc}; +use log::{debug, error, info}; +use std::collections::HashMap; +use tokio_util::sync::CancellationToken; + +pub fn environment_building_stdio_directory(working_directory: &Utf8Path) -> Utf8PathBuf { + working_directory.join("environment_building_stdio") +} + +pub fn build_environments(global_config: &GlobalConfig, suites: Vec) -> Result> { + let mut build_stage_reporter = BuildStageReporter::new( + suites.iter().map(|suite| suite.id.as_ref()), + &global_config.results_directory, + &global_config.results_directory_locker, + )?; + let env_building_stdio_directory = + environment_building_stdio_directory(&global_config.working_directory); + + let mut completed_suites = Vec::new(); + for suite in suites.into_iter() { + let outcome = build_environment( + &suite.id, + &suite.environment, + &global_config.cancellation_token, + &mut build_stage_reporter, + &env_building_stdio_directory, + )?; + match outcome { + BuildOutcome::NotNeeded => completed_suites.push(suite), + BuildOutcome::Success(_) => completed_suites.push(suite), + BuildOutcome::Terminated => bail!("Terminated"), + _ => {} + } + } + Ok(completed_suites) +} + +fn build_environment( + id: &str, + environment: &Environment, + cancellation_token: &CancellationToken, + build_stage_reporter: &mut BuildStageReporter, + stdio_directory: &Utf8Path, +) -> Result { + let Environment::Rcc(environment) = environment else { + let outcome = BuildOutcome::NotNeeded; + debug!("Nothing to do for suite {}", id); + build_stage_reporter.update(id, EnvironmentBuildStage::Complete(outcome.clone()))?; + return Ok(outcome); + }; + info!("Building environment for suite {}", id); + let command_spec = create_build_command(environment); + let supervisor = ChildProcessSupervisor { + command_spec: &command_spec, + stdio_paths: Some(StdioPaths { + stdout: stdio_directory.join(format!("{}.stdout", id)), + stderr: stdio_directory.join(format!("{}.stderr", id)), + }), + timeout: environment.build_timeout, + cancellation_token, + }; + let start_time = Utc::now(); + build_stage_reporter.update( + id, + EnvironmentBuildStage::InProgress(start_time.timestamp()), + )?; + let outcome = run_build_command(supervisor, start_time)?; + build_stage_reporter.update(id, EnvironmentBuildStage::Complete(outcome.clone()))?; + Ok(outcome) +} + +fn create_build_command(environment: &RCCEnvironment) -> CommandSpec { + let mut command_spec = CommandSpec::new(&environment.binary_path); + command_spec + .add_argument("holotree") + .add_argument("variables") + .add_argument("--json"); + apply_current_settings( + &environment.robot_yaml_path, + &environment.controller, + &environment.space, + environment.env_json_path.as_deref(), + &mut command_spec, + ); + command_spec +} + +fn run_build_command( + build_process_supervisor: ChildProcessSupervisor, + reference_timestamp_for_duration: DateTime, +) -> Result { + let build_result = build_process_supervisor.run(); + let child_process_outcome = match build_result { + Ok(o) => o, + Err(e) => { + let e = e.context("Environment building failed, suite will be dropped"); + let e = log_and_return_error(e); + return Ok(BuildOutcome::Error(format!("{e:?}"))); + } + }; + let duration = (Utc::now() - reference_timestamp_for_duration).num_seconds(); + match child_process_outcome { + ChildProcessOutcome::Exited(exit_status) => { + if exit_status.success() { + debug!("Environmenent building succeeded"); + Ok(BuildOutcome::Success(duration)) + } else { + error!("Environment building not sucessful, suite will be dropped"); + Ok(BuildOutcome::NonZeroExit) + } + } + ChildProcessOutcome::TimedOut => { + error!("Environment building timed out, suite will be dropped"); + Ok(BuildOutcome::Timeout) + } + ChildProcessOutcome::Terminated => Ok(BuildOutcome::Terminated), + } +} + +struct BuildStageReporter<'a> { + build_states: HashMap, + path: Utf8PathBuf, + locker: &'a Locker, +} + +impl<'a> BuildStageReporter<'a> { + pub fn new<'c>( + ids: impl Iterator, + results_directory: &Utf8Path, + locker: &'a Locker, + ) -> Result> { + let build_states: HashMap<_, _> = ids + .map(|id| (id.to_string(), EnvironmentBuildStage::Pending)) + .collect(); + let path = results_directory.join("environment_build_states.json"); + BuildStates(&build_states).write(&path, locker)?; + Ok(Self { + build_states, + path, + locker, + }) + } + + pub fn update(&mut self, suite_id: &str, build_status: EnvironmentBuildStage) -> Result<()> { + self.build_states.insert(suite_id.into(), build_status); + BuildStates(&self.build_states).write(&self.path, self.locker) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rcc_build_command() { + let mut expected = CommandSpec::new("/bin/rcc"); + expected + .add_argument("holotree") + .add_argument("variables") + .add_argument("--json") + .add_argument("--robot") + .add_argument("/a/b/c/robot.yaml") + .add_argument("--controller") + .add_argument("robotmk") + .add_argument("--space") + .add_argument("my_suite"); + + assert_eq!( + create_build_command(&RCCEnvironment { + binary_path: Utf8PathBuf::from("/bin/rcc"), + robot_yaml_path: Utf8PathBuf::from("/a/b/c/robot.yaml"), + controller: String::from("robotmk"), + space: String::from("my_suite"), + build_timeout: 123, + env_json_path: None, + }), + expected + ) + } +} diff --git a/v2/robotmk/src/bin/scheduler/environment.rs b/v2/robotmk/src/bin/scheduler/environment.rs deleted file mode 100644 index b7323081..00000000 --- a/v2/robotmk/src/bin/scheduler/environment.rs +++ /dev/null @@ -1,378 +0,0 @@ -use super::child_process_supervisor::{ChildProcessOutcome, ChildProcessSupervisor, StdioPaths}; -use super::command_spec::CommandSpec; -use super::internal_config::{GlobalConfig, Suite}; -use super::logging::log_and_return_error; -use super::results::{ - EnvironmentBuildStatesAdministrator, EnvironmentBuildStatus, EnvironmentBuildStatusError, -}; -use robotmk::config::EnvironmentConfig; - -use anyhow::{bail, Context, Result}; -use camino::{Utf8Path, Utf8PathBuf}; -use chrono::Utc; -use log::{debug, error, info}; - -pub fn environment_building_stdio_directory(working_directory: &Utf8Path) -> Utf8PathBuf { - working_directory.join("environment_building_stdio") -} - -pub fn build_environments(global_config: &GlobalConfig, suites: Vec) -> Result> { - let mut environment_build_states_administrator = - EnvironmentBuildStatesAdministrator::new_with_pending( - &suites, - &global_config.results_directory, - &global_config.results_directory_locker, - )?; - let env_building_stdio_directory = - environment_building_stdio_directory(&global_config.working_directory); - - suites - .into_iter() - .filter_map(|suite| { - match build_environment( - suite, - &mut environment_build_states_administrator, - &env_building_stdio_directory, - ) { - Ok(None) => None, - Ok(Some(suite)) => Some(Ok(suite)), - Err(e) => Some(Err(e)), - } - }) - .collect() -} - -fn build_environment( - suite: Suite, - environment_build_states_administrator: &mut EnvironmentBuildStatesAdministrator, - stdio_directory: &Utf8Path, -) -> Result> { - let suite = match suite.environment.build_instructions() { - Some(build_instructions) => { - info!("Building environment for suite {}", suite.id); - let start_time = Utc::now().timestamp(); - environment_build_states_administrator - .update(&suite.id, EnvironmentBuildStatus::InProgress(start_time))?; - let environment_build_status = run_environment_build( - ChildProcessSupervisor { - command_spec: &build_instructions.command_spec, - stdio_paths: Some(StdioPaths { - stdout: stdio_directory.join(format!("{}.stdout", suite.id)), - stderr: stdio_directory.join(format!("{}.stderr", suite.id)), - }), - timeout: build_instructions.timeout, - cancellation_token: &suite.cancellation_token, - }, - start_time, - )?; - let drop_suite = matches!(environment_build_status, EnvironmentBuildStatus::Failure(_)); - environment_build_states_administrator.update(&suite.id, environment_build_status)?; - (!drop_suite).then_some(suite) - } - None => { - debug!("Nothing to do for suite {}", suite.id); - environment_build_states_administrator - .update(&suite.id, EnvironmentBuildStatus::NotNeeded)?; - Some(suite) - } - }; - Ok(suite) -} - -fn run_environment_build( - build_process_supervisor: ChildProcessSupervisor, - reference_timestamp_for_duration: i64, -) -> Result { - let child_process_outcome = match build_process_supervisor - .run() - .context("Environment building failed, suite will be dropped") - .map_err(log_and_return_error) - { - Ok(child_process_outcome) => child_process_outcome, - Err(error) => { - return Ok(EnvironmentBuildStatus::Failure( - EnvironmentBuildStatusError::Error(format!("{error:?}")), - )) - } - }; - let duration = Utc::now().timestamp() - reference_timestamp_for_duration; - match child_process_outcome { - ChildProcessOutcome::Exited(exit_status) => { - if exit_status.success() { - debug!("Environmenent building succeeded"); - Ok(EnvironmentBuildStatus::Success(duration)) - } else { - error!("Environment building not sucessful, suite will be dropped"); - Ok(EnvironmentBuildStatus::Failure( - EnvironmentBuildStatusError::NonZeroExit, - )) - } - } - ChildProcessOutcome::TimedOut => { - error!("Environment building timed out, suite will be dropped"); - Ok(EnvironmentBuildStatus::Failure( - EnvironmentBuildStatusError::Timeout, - )) - } - ChildProcessOutcome::Terminated => { - bail!("Terminated") - } - } -} - -#[derive(Clone)] -#[cfg_attr(test, derive(Debug, PartialEq))] -pub enum Environment { - System(SystemEnvironment), - Rcc(RCCEnvironment), -} - -#[derive(Clone)] -#[cfg_attr(test, derive(Debug, PartialEq))] -pub struct SystemEnvironment {} - -#[derive(Clone)] -#[cfg_attr(test, derive(Debug, PartialEq))] -pub struct RCCEnvironment { - pub binary_path: Utf8PathBuf, - pub robot_yaml_path: Utf8PathBuf, - pub controller: String, - pub space: String, - pub build_timeout: u64, - pub env_json_path: Option, -} - -impl Environment { - pub fn new( - suite_id: &str, - rcc_binary_path: &Utf8Path, - environment_config: &EnvironmentConfig, - ) -> Self { - match environment_config { - EnvironmentConfig::System => Self::System(SystemEnvironment {}), - EnvironmentConfig::Rcc(rcc_environment_config) => Self::Rcc(RCCEnvironment { - binary_path: rcc_binary_path.to_path_buf(), - robot_yaml_path: rcc_environment_config.robot_yaml_path.clone(), - controller: String::from("robotmk"), - space: suite_id.to_string(), - build_timeout: rcc_environment_config.build_timeout, - env_json_path: rcc_environment_config.env_json_path.clone(), - }), - } - } - - pub fn wrap(&self, command_spec: CommandSpec) -> CommandSpec { - match self { - Self::System(system_environment) => system_environment.wrap(command_spec), - Self::Rcc(rcc_environment) => rcc_environment.wrap(command_spec), - } - } - - pub fn create_result_code(&self, exit_code: i32) -> ResultCode { - match self { - Self::System(_) => SystemEnvironment::create_result_code(exit_code), - Self::Rcc(_) => RCCEnvironment::create_result_code(exit_code), - } - } - - fn build_instructions(&self) -> Option { - match self { - Self::System(system_environment) => system_environment.build_instructions(), - Self::Rcc(rcc_environment) => rcc_environment.build_instructions(), - } - } -} - -pub enum ResultCode { - AllTestsPassed, - RobotCommandFailed, - EnvironmentFailed, -} - -struct BuildInstructions { - command_spec: CommandSpec, - timeout: u64, -} - -impl SystemEnvironment { - fn wrap(&self, command_spec: CommandSpec) -> CommandSpec { - command_spec - } - - fn create_result_code(exit_code: i32) -> ResultCode { - if exit_code == 0 { - return ResultCode::AllTestsPassed; - } - ResultCode::RobotCommandFailed - } - - fn build_instructions(&self) -> Option { - None - } -} - -impl RCCEnvironment { - fn wrap(&self, command_spec: CommandSpec) -> CommandSpec { - let mut wrapped_spec = CommandSpec::new(&self.binary_path); - wrapped_spec - .add_argument("task") - .add_argument("script") - .add_argument("--no-build"); - self.apply_current_settings(&mut wrapped_spec); - wrapped_spec - .add_argument("--") - .add_argument(command_spec.executable) - .add_arguments(command_spec.arguments); - wrapped_spec - } - - fn create_result_code(exit_code: i32) -> ResultCode { - match exit_code { - 0 => ResultCode::AllTestsPassed, - 10 => ResultCode::RobotCommandFailed, - _ => ResultCode::EnvironmentFailed, - } - } - - fn build_instructions(&self) -> Option { - let mut command_spec = CommandSpec::new(&self.binary_path); - self.apply_current_settings( - command_spec - .add_argument("holotree") - .add_argument("variables") - .add_argument("--json"), - ); - Some(BuildInstructions { - command_spec, - timeout: self.build_timeout, - }) - } - - fn apply_current_settings(&self, command_spec: &mut CommandSpec) { - command_spec - .add_argument("--robot") - .add_argument(&self.robot_yaml_path) - .add_argument("--controller") - .add_argument(&self.controller) - .add_argument("--space") - .add_argument(&self.space); - if let Some(env_json_path) = &self.env_json_path { - command_spec - .add_argument("--environment") - .add_argument(env_json_path); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use robotmk::config::RCCEnvironmentConfig; - - #[test] - fn environment_from_system_config() { - assert!( - Environment::new("my_suite", "/bin/rcc".into(), &EnvironmentConfig::System) - .build_instructions() - .is_none() - ) - } - - #[test] - fn environment_from_rcc_config() { - assert!(Environment::new( - "my_suite", - "/bin/rcc".into(), - &EnvironmentConfig::Rcc(RCCEnvironmentConfig { - robot_yaml_path: Utf8PathBuf::from("/a/b/c/robot.yaml"), - build_timeout: 60, - env_json_path: None, - }) - ) - .build_instructions() - .is_some()) - } - - fn command_spec_for_wrap() -> CommandSpec { - let mut command_spec = CommandSpec::new("C:\\x\\y\\z.exe"); - command_spec - .add_argument("arg1") - .add_argument("--flag") - .add_argument("--option") - .add_argument("option_value"); - command_spec - } - - #[test] - fn test_system_wrap() { - assert_eq!( - SystemEnvironment {}.wrap(command_spec_for_wrap()), - command_spec_for_wrap() - ); - } - - #[test] - fn test_rcc_wrap() { - let mut expected = CommandSpec::new("C:\\bin\\z.exe"); - expected - .add_argument("task") - .add_argument("script") - .add_argument("--no-build") - .add_argument("--robot") - .add_argument("C:\\my_suite\\robot.yaml") - .add_argument("--controller") - .add_argument("robotmk") - .add_argument("--space") - .add_argument("my_suite") - .add_argument("--environment") - .add_argument("C:\\my_suite\\env.json") - .add_argument("--") - .add_argument("C:\\x\\y\\z.exe") - .add_argument("arg1") - .add_argument("--flag") - .add_argument("--option") - .add_argument("option_value"); - assert_eq!( - RCCEnvironment { - binary_path: Utf8PathBuf::from("C:\\bin\\z.exe"), - robot_yaml_path: Utf8PathBuf::from("C:\\my_suite\\robot.yaml"), - controller: String::from("robotmk"), - space: String::from("my_suite"), - build_timeout: 600, - env_json_path: Some("C:\\my_suite\\env.json".into()) - } - .wrap(command_spec_for_wrap()), - expected - ); - } - - #[test] - fn rcc_build_command() { - let mut expected = CommandSpec::new("/bin/rcc"); - expected - .add_argument("holotree") - .add_argument("variables") - .add_argument("--json") - .add_argument("--robot") - .add_argument("/a/b/c/robot.yaml") - .add_argument("--controller") - .add_argument("robotmk") - .add_argument("--space") - .add_argument("my_suite"); - - assert_eq!( - RCCEnvironment { - binary_path: Utf8PathBuf::from("/bin/rcc"), - robot_yaml_path: Utf8PathBuf::from("/a/b/c/robot.yaml"), - controller: String::from("robotmk"), - space: String::from("my_suite"), - build_timeout: 123, - env_json_path: None, - } - .build_instructions() - .unwrap() - .command_spec, - expected - ) - } -} diff --git a/v2/robotmk/src/bin/scheduler/internal_config.rs b/v2/robotmk/src/bin/scheduler/internal_config.rs index 52c9d531..c69ec0e6 100644 --- a/v2/robotmk/src/bin/scheduler/internal_config.rs +++ b/v2/robotmk/src/bin/scheduler/internal_config.rs @@ -1,12 +1,10 @@ -use crate::environment::Environment; -use crate::results::suite_results_directory; -use crate::rf::robot::Robot; -use crate::sessions::session::Session; -use robotmk::{ - config::{Config, RCCConfig, WorkingDirectoryCleanupConfig}, - lock::Locker, - section::Host, -}; +use robotmk::config::{Config, RCCConfig, WorkingDirectoryCleanupConfig}; +use robotmk::environment::Environment; +use robotmk::lock::Locker; +use robotmk::results::suite_results_directory; +use robotmk::rf::robot::Robot; +use robotmk::section::Host; +use robotmk::sessions::session::Session; use camino::Utf8PathBuf; use tokio_util::sync::CancellationToken; @@ -91,12 +89,12 @@ pub fn sort_suites_by_id(suites: &mut [Suite]) { #[cfg(test)] mod tests { use super::*; - use crate::environment::{RCCEnvironment, SystemEnvironment}; - use crate::sessions::session::{CurrentSession, UserSession}; use robotmk::config::{ EnvironmentConfig, ExecutionConfig, RCCEnvironmentConfig, RCCProfileConfig, RetryStrategy, RobotFrameworkConfig, SessionConfig, SuiteConfig, UserSessionConfig, }; + use robotmk::environment::{Environment, RCCEnvironment, SystemEnvironment}; + use robotmk::sessions::session::{CurrentSession, UserSession}; use std::collections::HashMap; diff --git a/v2/robotmk/src/bin/scheduler/main.rs b/v2/robotmk/src/bin/scheduler/main.rs index 7b1e68f8..393921fe 100644 --- a/v2/robotmk/src/bin/scheduler/main.rs +++ b/v2/robotmk/src/bin/scheduler/main.rs @@ -1,13 +1,8 @@ -mod child_process_supervisor; +mod build; mod cli; -mod command_spec; -mod environment; mod internal_config; mod logging; -mod results; -mod rf; mod scheduling; -mod sessions; mod setup; mod termination; @@ -16,6 +11,7 @@ use clap::Parser; use log::{debug, info}; use logging::log_and_return_error; use robotmk::lock::Locker; +use robotmk::results::SchedulerPhase; use robotmk::section::WriteSection; fn main() -> Result<()> { @@ -48,7 +44,7 @@ fn run() -> Result<()> { setup::general::setup(&global_config, &suites).context("General setup failed")?; debug!("General setup completed"); - write_phase(&results::SchedulerPhase::RCCSetup, &global_config)?; + write_phase(&SchedulerPhase::RCCSetup, &global_config)?; let suites = setup::rcc::setup(&global_config, suites).context("RCC-specific setup failed")?; debug!("RCC-specific setup completed"); @@ -57,11 +53,8 @@ fn run() -> Result<()> { } info!("Starting environment building"); - write_phase( - &results::SchedulerPhase::EnvironmentBuilding, - &global_config, - )?; - let suites = environment::build_environments(&global_config, suites)?; + write_phase(&SchedulerPhase::EnvironmentBuilding, &global_config)?; + let suites = build::build_environments(&global_config, suites)?; info!("Environment building finished"); if global_config.cancellation_token.is_cancelled() { @@ -69,12 +62,12 @@ fn run() -> Result<()> { } info!("Starting suite scheduling"); - write_phase(&results::SchedulerPhase::Scheduling, &global_config)?; + write_phase(&SchedulerPhase::Scheduling, &global_config)?; scheduling::scheduler::run_suites_and_cleanup(&global_config, &suites) } fn write_phase( - phase: &results::SchedulerPhase, + phase: &SchedulerPhase, global_config: &internal_config::GlobalConfig, ) -> Result<()> { phase.write( diff --git a/v2/robotmk/src/bin/scheduler/scheduling.rs b/v2/robotmk/src/bin/scheduler/scheduling/mod.rs similarity index 100% rename from v2/robotmk/src/bin/scheduler/scheduling.rs rename to v2/robotmk/src/bin/scheduler/scheduling/mod.rs diff --git a/v2/robotmk/src/bin/scheduler/scheduling/suites.rs b/v2/robotmk/src/bin/scheduler/scheduling/suites.rs index aeeac009..35a9e455 100644 --- a/v2/robotmk/src/bin/scheduler/scheduling/suites.rs +++ b/v2/robotmk/src/bin/scheduler/scheduling/suites.rs @@ -1,13 +1,10 @@ -use crate::environment::ResultCode; use crate::internal_config::Suite; -use crate::results::{AttemptOutcome, AttemptsConfig, SuiteExecutionReport}; -use crate::rf::{rebot::Rebot, robot::Attempt}; -use crate::sessions::session::{RunOutcome, RunSpec}; +use robotmk::results::{AttemptsConfig, SuiteExecutionReport}; +use robotmk::suites::run_attempts_with_rebot; -use anyhow::{bail, Context, Result}; -use camino::{Utf8Path, Utf8PathBuf}; +use anyhow::{Context, Result}; use chrono::Utc; -use log::{debug, error}; +use log::debug; use robotmk::section::WritePiggybackSection; use std::fs::create_dir_all; @@ -35,24 +32,20 @@ fn produce_suite_results(suite: &Suite) -> Result { output_directory ))?; - let (attempt_outcomes, output_paths) = run_attempts_until_succesful(suite, &output_directory)?; + let (attempt_outcomes, rebot) = run_attempts_with_rebot( + &suite.robot, + &suite.id, + &suite.environment, + &suite.session, + suite.timeout, + &suite.cancellation_token, + &output_directory, + )?; Ok(SuiteExecutionReport { suite_id: suite.id.clone(), attempts: attempt_outcomes, - rebot: if output_paths.is_empty() { - None - } else { - Some( - Rebot { - environment: &suite.environment, - input_paths: &output_paths, - path_xml: &output_directory.join("rebot.xml"), - path_html: &output_directory.join("rebot.html"), - } - .rebot(), - ) - }, + rebot, config: AttemptsConfig { interval: suite.execution_interval_seconds, timeout: suite.timeout, @@ -60,89 +53,3 @@ fn produce_suite_results(suite: &Suite) -> Result { }, }) } - -fn run_attempts_until_succesful( - suite: &Suite, - output_directory: &Utf8Path, -) -> Result<(Vec, Vec)> { - let mut outcomes = vec![]; - let mut output_paths: Vec = vec![]; - - for attempt in suite.robot.attempts(output_directory) { - let (outcome, output_path) = run_attempt(suite, attempt, output_directory)?; - let success = matches!(&outcome, &AttemptOutcome::AllTestsPassed); - outcomes.push(outcome); - if let Some(output_path) = output_path { - output_paths.push(output_path); - } - if success { - break; - } - } - - Ok((outcomes, output_paths)) -} - -fn run_attempt( - suite: &Suite, - attempt: Attempt, - output_directory: &Utf8Path, -) -> Result<(AttemptOutcome, Option)> { - let log_message_start = format!("Suite {}, attempt {}", suite.id, attempt.index); - - let run_outcome = match suite.session.run(&RunSpec { - id: &format!("robotmk_suite_{}_attempt_{}", suite.id, attempt.index), - command_spec: &suite.environment.wrap(attempt.command_spec), - base_path: &output_directory.join(attempt.index.to_string()), - timeout: suite.timeout, - cancellation_token: &suite.cancellation_token, - }) { - Ok(run_outcome) => run_outcome, - Err(error_) => { - error!("{log_message_start}: {error_:?}"); - return Ok((AttemptOutcome::OtherError(format!("{error_:?}")), None)); - } - }; - let exit_code = match run_outcome { - RunOutcome::Exited(exit_code) => exit_code, - RunOutcome::TimedOut => { - error!("{log_message_start}: timed out"); - return Ok((AttemptOutcome::TimedOut, None)); - } - RunOutcome::Terminated => bail!("Terminated"), - }; - let exit_code = match exit_code { - Some(exit_code) => exit_code, - None => { - error!("{log_message_start}: failed to query exit code"); - return Ok(( - AttemptOutcome::OtherError( - "Failed to query exit code of Robot Framework call".into(), - ), - None, - )); - } - }; - match suite.environment.create_result_code(exit_code) { - ResultCode::AllTestsPassed => { - debug!("{log_message_start}: all tests passed"); - Ok(( - AttemptOutcome::AllTestsPassed, - Some(attempt.output_xml_file), - )) - } - ResultCode::EnvironmentFailed => { - error!("{log_message_start}: environment failure"); - Ok((AttemptOutcome::EnvironmentFailure, None)) - } - ResultCode::RobotCommandFailed => { - if attempt.output_xml_file.exists() { - debug!("{log_message_start}: some tests failed"); - Ok((AttemptOutcome::TestFailures, Some(attempt.output_xml_file))) - } else { - error!("{log_message_start}: Robot Framework failure (no output)"); - Ok((AttemptOutcome::RobotFrameworkFailure, None)) - } - } - } -} diff --git a/v2/robotmk/src/bin/scheduler/setup/general.rs b/v2/robotmk/src/bin/scheduler/setup/general.rs index f21cc0d2..2f2189c9 100644 --- a/v2/robotmk/src/bin/scheduler/setup/general.rs +++ b/v2/robotmk/src/bin/scheduler/setup/general.rs @@ -1,10 +1,10 @@ use super::icacls::run_icacls_command; -use crate::environment::environment_building_stdio_directory; +use crate::build::environment_building_stdio_directory; use crate::internal_config::{GlobalConfig, Suite}; -use crate::results::suite_results_directory; use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use log::debug; +use robotmk::results::suite_results_directory; use std::collections::HashSet; use std::fs::{create_dir_all, remove_file}; diff --git a/v2/robotmk/src/bin/scheduler/setup.rs b/v2/robotmk/src/bin/scheduler/setup/mod.rs similarity index 100% rename from v2/robotmk/src/bin/scheduler/setup.rs rename to v2/robotmk/src/bin/scheduler/setup/mod.rs diff --git a/v2/robotmk/src/bin/scheduler/setup/rcc.rs b/v2/robotmk/src/bin/scheduler/setup/rcc.rs index dc08ae2a..8813d566 100644 --- a/v2/robotmk/src/bin/scheduler/setup/rcc.rs +++ b/v2/robotmk/src/bin/scheduler/setup/rcc.rs @@ -1,10 +1,10 @@ use super::icacls::run_icacls_command; -use crate::command_spec::CommandSpec; -use crate::environment::Environment; use crate::internal_config::{sort_suites_by_id, GlobalConfig, Suite}; use crate::logging::log_and_return_error; -use crate::results::RCCSetupFailures; -use crate::sessions::session::{CurrentSession, RunOutcome, RunSpec, Session}; +use robotmk::command_spec::CommandSpec; +use robotmk::environment::Environment; +use robotmk::results::RCCSetupFailures; +use robotmk::sessions::session::{CurrentSession, RunOutcome, RunSpec, Session}; use anyhow::{bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; diff --git a/v2/robotmk/src/bin/scheduler/termination.rs b/v2/robotmk/src/bin/scheduler/termination.rs index 0b99b390..0784c716 100644 --- a/v2/robotmk/src/bin/scheduler/termination.rs +++ b/v2/robotmk/src/bin/scheduler/termination.rs @@ -1,10 +1,8 @@ use anyhow::{Context, Result}; use camino::Utf8PathBuf; use log::debug; -use std::collections::{HashMap, HashSet}; use std::thread::{sleep, spawn}; use std::time::Duration; -use sysinfo::{Pid, Process, ProcessExt, System, SystemExt}; use tokio_util::sync::CancellationToken; pub fn start_termination_control(run_flag_file: Option) -> Result { @@ -31,53 +29,6 @@ fn start_run_flag_watch_thread(file: Utf8PathBuf, token: CancellationToken) { }); } -// This is a non-cooperative termination (SIGKILL) of the entire process tree. What we would -// actually like to do is to shut down our child co-operatively and leave the termination of any -// non-direct children further down the tree to our child. However, Windows offers no API for this -// (there is no SIGTERM on Windows), so we instead kill the entire tree. -pub fn kill_process_tree(top_pid: &Pid) { - let mut system = System::new_all(); - system.refresh_processes(); - let processes = system.processes(); - - match processes.get(top_pid) { - None => return, - Some(top_process) => top_process.kill(), - }; - - kill_all_children(top_pid, processes); -} - -fn kill_all_children<'a>(top_pid: &'a Pid, processes: &'a HashMap) { - let mut pids_in_tree = HashSet::from([top_pid]); - - loop { - let current_tree_size = pids_in_tree.len(); - add_and_kill_direct_children(&mut pids_in_tree, processes); - if pids_in_tree.len() == current_tree_size { - break; - } - } -} - -fn add_and_kill_direct_children<'a>( - pids_in_tree: &mut HashSet<&'a Pid>, - processes: &'a HashMap, -) { - for (pid, parent_pid, process) in processes.iter().filter_map(|(pid, process)| { - process - .parent() - .map(|parent_pid| (pid, parent_pid, process)) - }) { - { - if pids_in_tree.contains(&parent_pid) { - pids_in_tree.insert(pid); - process.kill(); - } - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/v2/robotmk/src/bin/scheduler/child_process_supervisor.rs b/v2/robotmk/src/child_process_supervisor.rs similarity index 95% rename from v2/robotmk/src/bin/scheduler/child_process_supervisor.rs rename to v2/robotmk/src/child_process_supervisor.rs index fa187915..e177c27e 100644 --- a/v2/robotmk/src/bin/scheduler/child_process_supervisor.rs +++ b/v2/robotmk/src/child_process_supervisor.rs @@ -1,6 +1,5 @@ -use super::command_spec::CommandSpec; -use super::termination::kill_process_tree; -use robotmk::termination::{waited, Outcome}; +use crate::command_spec::CommandSpec; +use crate::termination::{kill_process_tree, waited, Outcome}; use anyhow::{Context, Result}; use camino::Utf8PathBuf; diff --git a/v2/robotmk/src/bin/scheduler/command_spec.rs b/v2/robotmk/src/command_spec.rs similarity index 98% rename from v2/robotmk/src/bin/scheduler/command_spec.rs rename to v2/robotmk/src/command_spec.rs index 89b6bf22..3b7583f5 100644 --- a/v2/robotmk/src/bin/scheduler/command_spec.rs +++ b/v2/robotmk/src/command_spec.rs @@ -2,7 +2,7 @@ use std::convert::From; use std::fmt::{Display, Formatter, Result}; use std::process::Command; -#[cfg_attr(test, derive(Debug, PartialEq))] +#[derive(Debug, PartialEq)] pub struct CommandSpec { pub executable: String, pub arguments: Vec, diff --git a/v2/robotmk/src/environment.rs b/v2/robotmk/src/environment.rs new file mode 100644 index 00000000..e2eee613 --- /dev/null +++ b/v2/robotmk/src/environment.rs @@ -0,0 +1,185 @@ +use crate::command_spec::CommandSpec; +use crate::config::EnvironmentConfig; + +use camino::{Utf8Path, Utf8PathBuf}; + +pub enum ResultCode { + AllTestsPassed, + RobotCommandFailed, + EnvironmentFailed, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Environment { + System(SystemEnvironment), + Rcc(RCCEnvironment), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SystemEnvironment {} + +#[derive(Clone, Debug, PartialEq)] +pub struct RCCEnvironment { + pub binary_path: Utf8PathBuf, + pub robot_yaml_path: Utf8PathBuf, + pub controller: String, + pub space: String, + pub build_timeout: u64, + pub env_json_path: Option, +} + +impl Environment { + pub fn new( + suite_id: &str, + rcc_binary_path: &Utf8Path, + environment_config: &EnvironmentConfig, + ) -> Self { + match environment_config { + EnvironmentConfig::System => Self::System(SystemEnvironment {}), + EnvironmentConfig::Rcc(rcc_environment_config) => Self::Rcc(RCCEnvironment { + binary_path: rcc_binary_path.to_path_buf(), + robot_yaml_path: rcc_environment_config.robot_yaml_path.clone(), + controller: String::from("robotmk"), + space: suite_id.to_string(), + build_timeout: rcc_environment_config.build_timeout, + env_json_path: rcc_environment_config.env_json_path.clone(), + }), + } + } + + pub fn wrap(&self, command_spec: CommandSpec) -> CommandSpec { + match self { + Self::System(system_environment) => system_environment.wrap(command_spec), + Self::Rcc(rcc_environment) => rcc_environment.wrap(command_spec), + } + } + + pub fn create_result_code(&self, exit_code: i32) -> ResultCode { + match self { + Self::System(_) => SystemEnvironment::create_result_code(exit_code), + Self::Rcc(_) => RCCEnvironment::create_result_code(exit_code), + } + } +} + +impl SystemEnvironment { + fn wrap(&self, command_spec: CommandSpec) -> CommandSpec { + command_spec + } + + fn create_result_code(exit_code: i32) -> ResultCode { + if exit_code == 0 { + return ResultCode::AllTestsPassed; + } + ResultCode::RobotCommandFailed + } +} + +impl RCCEnvironment { + fn wrap(&self, command_spec: CommandSpec) -> CommandSpec { + let mut wrapped_spec = CommandSpec::new(&self.binary_path); + wrapped_spec + .add_argument("task") + .add_argument("script") + .add_argument("--no-build"); + apply_current_settings( + &self.robot_yaml_path, + &self.controller, + &self.space, + self.env_json_path.as_deref(), + &mut wrapped_spec, + ); + wrapped_spec + .add_argument("--") + .add_argument(command_spec.executable) + .add_arguments(command_spec.arguments); + wrapped_spec + } + + fn create_result_code(exit_code: i32) -> ResultCode { + match exit_code { + 0 => ResultCode::AllTestsPassed, + 10 => ResultCode::RobotCommandFailed, + _ => ResultCode::EnvironmentFailed, + } + } +} + +pub fn apply_current_settings( + robot_yaml_path: &Utf8Path, + controller: &str, + space: &str, + env_json_path: Option<&Utf8Path>, + command_spec: &mut CommandSpec, +) { + command_spec + .add_argument("--robot") + .add_argument(robot_yaml_path) + .add_argument("--controller") + .add_argument(controller) + .add_argument("--space") + .add_argument(space); + if let Some(env_json_path) = &env_json_path { + command_spec + .add_argument("--environment") + .add_argument(env_json_path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn command_spec_for_wrap() -> CommandSpec { + let mut command_spec = CommandSpec::new("C:\\x\\y\\z.exe"); + command_spec + .add_argument("arg1") + .add_argument("--flag") + .add_argument("--option") + .add_argument("option_value"); + command_spec + } + + #[test] + fn test_system_wrap() { + assert_eq!( + SystemEnvironment {}.wrap(command_spec_for_wrap()), + command_spec_for_wrap() + ); + } + + #[test] + fn test_rcc_wrap() { + let mut expected = CommandSpec::new("C:\\bin\\z.exe"); + expected + .add_argument("task") + .add_argument("script") + .add_argument("--no-build") + .add_argument("--robot") + .add_argument("C:\\my_suite\\robot.yaml") + .add_argument("--controller") + .add_argument("robotmk") + .add_argument("--space") + .add_argument("my_suite") + .add_argument("--environment") + .add_argument("C:\\my_suite\\env.json") + .add_argument("--") + .add_argument("C:\\x\\y\\z.exe") + .add_argument("arg1") + .add_argument("--flag") + .add_argument("--option") + .add_argument("option_value"); + assert_eq!( + RCCEnvironment { + binary_path: Utf8PathBuf::from("C:\\bin\\z.exe"), + robot_yaml_path: Utf8PathBuf::from("C:\\my_suite\\robot.yaml"), + controller: String::from("robotmk"), + space: String::from("my_suite"), + build_timeout: 600, + env_json_path: Some("C:\\my_suite\\env.json".into()) + } + .wrap(command_spec_for_wrap()), + expected + ); + } +} diff --git a/v2/robotmk/src/lib.rs b/v2/robotmk/src/lib.rs index 2b96875a..5a934458 100644 --- a/v2/robotmk/src/lib.rs +++ b/v2/robotmk/src/lib.rs @@ -1,4 +1,11 @@ +pub mod child_process_supervisor; +pub mod command_spec; pub mod config; +pub mod environment; pub mod lock; +pub mod results; +pub mod rf; pub mod section; +pub mod sessions; +pub mod suites; pub mod termination; diff --git a/v2/robotmk/src/bin/scheduler/results.rs b/v2/robotmk/src/results.rs similarity index 58% rename from v2/robotmk/src/bin/scheduler/results.rs rename to v2/robotmk/src/results.rs index 77b2e5d6..f49c19f7 100644 --- a/v2/robotmk/src/bin/scheduler/results.rs +++ b/v2/robotmk/src/results.rs @@ -1,10 +1,5 @@ -use super::internal_config::Suite; -use anyhow::Result; +use crate::section::{WritePiggybackSection, WriteSection}; use camino::{Utf8Path, Utf8PathBuf}; -use robotmk::{ - lock::Locker, - section::{WritePiggybackSection, WriteSection}, -}; use serde::Serialize; use std::collections::HashMap; @@ -40,14 +35,8 @@ impl WriteSection for RCCSetupFailures { } } -pub struct EnvironmentBuildStatesAdministrator<'a> { - build_states: HashMap, - path: Utf8PathBuf, - locker: &'a Locker, -} - #[derive(Serialize)] -pub struct BuildStates<'a>(&'a HashMap); +pub struct BuildStates<'a>(pub &'a HashMap); impl WriteSection for BuildStates<'_> { fn name() -> &'static str { @@ -55,45 +44,21 @@ impl WriteSection for BuildStates<'_> { } } -impl<'a> EnvironmentBuildStatesAdministrator<'a> { - pub fn new_with_pending( - suites: &[Suite], - results_directory: &Utf8Path, - locker: &'a Locker, - ) -> Result> { - let build_states: HashMap<_, _> = suites - .iter() - .map(|suite| (suite.id.to_string(), EnvironmentBuildStatus::Pending)) - .collect(); - let path = results_directory.join("environment_build_states.json"); - BuildStates(&build_states).write(&path, locker)?; - Ok(Self { - build_states, - path, - locker, - }) - } - - pub fn update(&mut self, suite_id: &str, build_status: EnvironmentBuildStatus) -> Result<()> { - self.build_states.insert(suite_id.into(), build_status); - BuildStates(&self.build_states).write(&self.path, self.locker) - } +#[derive(PartialEq, Debug, Serialize, Clone)] +pub enum BuildOutcome { + NotNeeded, + Success(i64), + NonZeroExit, + Timeout, + Terminated, + Error(String), } #[derive(Serialize)] -pub enum EnvironmentBuildStatus { - Success(i64), - Failure(EnvironmentBuildStatusError), - NotNeeded, +pub enum EnvironmentBuildStage { Pending, InProgress(i64), -} - -#[derive(Serialize)] -pub enum EnvironmentBuildStatusError { - NonZeroExit, - Timeout, - Error(String), + Complete(BuildOutcome), } #[derive(Serialize)] diff --git a/v2/robotmk/src/bin/scheduler/rf.rs b/v2/robotmk/src/rf/mod.rs similarity index 100% rename from v2/robotmk/src/bin/scheduler/rf.rs rename to v2/robotmk/src/rf/mod.rs diff --git a/v2/robotmk/src/bin/scheduler/rf/rebot.rs b/v2/robotmk/src/rf/rebot.rs similarity index 97% rename from v2/robotmk/src/bin/scheduler/rf/rebot.rs rename to v2/robotmk/src/rf/rebot.rs index 7003fe6d..546d3eb6 100644 --- a/v2/robotmk/src/bin/scheduler/rf/rebot.rs +++ b/v2/robotmk/src/rf/rebot.rs @@ -1,6 +1,7 @@ use super::robot::PYTHON_EXECUTABLE; use crate::command_spec::CommandSpec; -use crate::environment::{Environment, ResultCode}; +use crate::environment::Environment; +use crate::environment::ResultCode; use crate::results::{RebotOutcome, RebotResult}; use anyhow::{Context, Result}; @@ -110,7 +111,7 @@ impl Rebot<'_> { #[cfg(test)] mod tests { use super::*; - use robotmk::config::EnvironmentConfig; + use crate::config::EnvironmentConfig; #[test] fn build_rebot_command() { diff --git a/v2/robotmk/src/bin/scheduler/rf/robot.rs b/v2/robotmk/src/rf/robot.rs similarity index 98% rename from v2/robotmk/src/bin/scheduler/rf/robot.rs rename to v2/robotmk/src/rf/robot.rs index 8e43ece1..bc2e4e3a 100644 --- a/v2/robotmk/src/bin/scheduler/rf/robot.rs +++ b/v2/robotmk/src/rf/robot.rs @@ -1,12 +1,11 @@ use crate::command_spec::CommandSpec; -use robotmk::config::RetryStrategy; +use crate::config::RetryStrategy; use camino::{Utf8Path, Utf8PathBuf}; pub const PYTHON_EXECUTABLE: &str = "python"; -#[derive(Clone)] -#[cfg_attr(test, derive(Debug, PartialEq))] +#[derive(Clone, Debug, PartialEq)] pub struct Robot { pub robot_target: Utf8PathBuf, pub command_line_args: Vec, diff --git a/v2/robotmk/src/bin/scheduler/sessions.rs b/v2/robotmk/src/sessions/mod.rs similarity index 100% rename from v2/robotmk/src/bin/scheduler/sessions.rs rename to v2/robotmk/src/sessions/mod.rs diff --git a/v2/robotmk/src/bin/scheduler/sessions/schtasks.rs b/v2/robotmk/src/sessions/schtasks.rs similarity index 98% rename from v2/robotmk/src/bin/scheduler/sessions/schtasks.rs rename to v2/robotmk/src/sessions/schtasks.rs index 59c570ae..c3b089e3 100644 --- a/v2/robotmk/src/bin/scheduler/sessions/schtasks.rs +++ b/v2/robotmk/src/sessions/schtasks.rs @@ -1,8 +1,6 @@ use super::session::RunOutcome; use crate::command_spec::CommandSpec; -use crate::logging::log_and_return_error; -use crate::termination::kill_process_tree; -use robotmk::termination::{waited, Outcome}; +use crate::termination::{kill_process_tree, waited, Outcome}; use anyhow::{bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; @@ -243,7 +241,7 @@ fn kill_and_delete_task(task_name: &str, paths: &Paths) { fn kill_task(paths: &Paths) { let _ = remove_file(&paths.run_flag) .context(format!("Failed to remove {}", paths.run_flag)) - .map_err(log_and_return_error); + .map_err(|e| error!("{:?}", e)); let _ = kill_task_via_pid(&paths.pid).map_err(|error| { warn!("{:?}", error); error @@ -263,7 +261,7 @@ fn delete_task(task_name: &str) { debug!("Deleting task {task_name}"); let _ = run_schtasks(["/delete", "/tn", task_name, "/f"]) .context(format!("Failed to delete task {}", task_name)) - .map_err(log_and_return_error); + .map_err(|e| error!("{:?}", e)); } fn read_until_first_whitespace(path: &Utf8Path) -> Result { diff --git a/v2/robotmk/src/bin/scheduler/sessions/session.rs b/v2/robotmk/src/sessions/session.rs similarity index 93% rename from v2/robotmk/src/bin/scheduler/sessions/session.rs rename to v2/robotmk/src/sessions/session.rs index b5e6bb2f..0d34b654 100644 --- a/v2/robotmk/src/bin/scheduler/sessions/session.rs +++ b/v2/robotmk/src/sessions/session.rs @@ -1,15 +1,14 @@ use super::schtasks::{run_task, TaskSpec}; use crate::child_process_supervisor::{ChildProcessOutcome, ChildProcessSupervisor, StdioPaths}; use crate::command_spec::CommandSpec; -use robotmk::config::SessionConfig; +use crate::config::SessionConfig; use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; use std::fmt::{Display, Formatter, Result as FmtResult}; use tokio_util::sync::CancellationToken; -#[derive(Clone, Eq, Hash, PartialEq)] -#[cfg_attr(test, derive(Debug))] +#[derive(Clone, Eq, Hash, PartialEq, Debug)] pub enum Session { Current(CurrentSession), User(UserSession), @@ -46,8 +45,7 @@ impl Display for Session { } } -#[derive(Clone, Eq, Hash, PartialEq)] -#[cfg_attr(test, derive(Debug))] +#[derive(Clone, Eq, Hash, PartialEq, Debug)] pub struct CurrentSession {} impl Display for CurrentSession { @@ -56,8 +54,7 @@ impl Display for CurrentSession { } } -#[derive(Clone, Eq, Hash, PartialEq)] -#[cfg_attr(test, derive(Debug))] +#[derive(Clone, Eq, Hash, PartialEq, Debug)] pub struct UserSession { pub user_name: String, } diff --git a/v2/robotmk/src/suites.rs b/v2/robotmk/src/suites.rs new file mode 100644 index 00000000..cc2f42bc --- /dev/null +++ b/v2/robotmk/src/suites.rs @@ -0,0 +1,123 @@ +use crate::environment::{Environment, ResultCode}; +use crate::results::{AttemptOutcome, RebotOutcome}; +use crate::rf::rebot::Rebot; +use crate::rf::robot::{Attempt, Robot}; +use crate::sessions::session::{RunOutcome, RunSpec, Session}; +use anyhow::{bail, Result}; +use camino::{Utf8Path, Utf8PathBuf}; +use log::{debug, error}; +use tokio_util::sync::CancellationToken; + +pub fn run_attempts_with_rebot( + robot: &Robot, + id: &str, + environment: &Environment, + session: &Session, + timeout: u64, + cancellation_token: &CancellationToken, + output_directory: &Utf8Path, +) -> Result<(Vec, Option)> { + let mut outcomes = vec![]; + let mut output_paths: Vec = vec![]; + + for attempt in robot.attempts(output_directory) { + let (outcome, output_path) = run_attempt( + id, + environment, + session, + timeout, + attempt, + cancellation_token, + output_directory, + )?; + let success = matches!(&outcome, &AttemptOutcome::AllTestsPassed); + outcomes.push(outcome); + if let Some(output_path) = output_path { + output_paths.push(output_path); + } + if success { + break; + } + } + + if output_paths.is_empty() { + return Ok((outcomes, None)); + } + let rebot = Rebot { + environment, + input_paths: &output_paths, + path_xml: &output_directory.join("rebot.xml"), + path_html: &output_directory.join("rebot.html"), + } + .rebot(); + + Ok((outcomes, Some(rebot))) +} + +fn run_attempt( + id: &str, + environment: &Environment, + session: &Session, + timeout: u64, + attempt: Attempt, + cancellation_token: &CancellationToken, + output_directory: &Utf8Path, +) -> Result<(AttemptOutcome, Option)> { + let log_message_start = format!("Suite {}, attempt {}", id, attempt.index); + + let run_outcome = match session.run(&RunSpec { + id: &format!("robotmk_suite_{}_attempt_{}", id, attempt.index), + command_spec: &environment.wrap(attempt.command_spec), + base_path: &output_directory.join(attempt.index.to_string()), + timeout, + cancellation_token, + }) { + Ok(run_outcome) => run_outcome, + Err(error_) => { + error!("{log_message_start}: {error_:?}"); + return Ok((AttemptOutcome::OtherError(format!("{error_:?}")), None)); + } + }; + let exit_code = match run_outcome { + RunOutcome::Exited(exit_code) => exit_code, + RunOutcome::TimedOut => { + error!("{log_message_start}: timed out"); + return Ok((AttemptOutcome::TimedOut, None)); + } + RunOutcome::Terminated => bail!("Terminated"), + }; + let exit_code = match exit_code { + Some(exit_code) => exit_code, + None => { + error!("{log_message_start}: failed to query exit code"); + return Ok(( + AttemptOutcome::OtherError( + "Failed to query exit code of Robot Framework call".into(), + ), + None, + )); + } + }; + match environment.create_result_code(exit_code) { + ResultCode::AllTestsPassed => { + debug!("{log_message_start}: all tests passed"); + Ok(( + AttemptOutcome::AllTestsPassed, + Some(attempt.output_xml_file), + )) + } + ResultCode::EnvironmentFailed => { + error!("{log_message_start}: environment failure"); + Ok((AttemptOutcome::EnvironmentFailure, None)) + } + ResultCode::RobotCommandFailed => { + if attempt.output_xml_file.exists() { + debug!("{log_message_start}: some tests failed"); + Ok((AttemptOutcome::TestFailures, Some(attempt.output_xml_file))) + } else { + error!("{log_message_start}: Robot Framework failure (no output)"); + Ok((AttemptOutcome::RobotFrameworkFailure, None)) + } + } + } +} diff --git a/v2/robotmk/src/termination.rs b/v2/robotmk/src/termination.rs index 5e287f96..cc0b70c3 100644 --- a/v2/robotmk/src/termination.rs +++ b/v2/robotmk/src/termination.rs @@ -1,5 +1,7 @@ +use std::collections::{HashMap, HashSet}; use std::future::Future; use std::time::Duration; +use sysinfo::{Pid, Process, ProcessExt, System, SystemExt}; use tokio::time::sleep; use tokio_util::sync::CancellationToken; @@ -24,3 +26,50 @@ where _ = sleep(duration) => { Outcome::Timeout }, } } + +// This is a non-cooperative termination (SIGKILL) of the entire process tree. What we would +// actually like to do is to shut down our child co-operatively and leave the termination of any +// non-direct children further down the tree to our child. However, Windows offers no API for this +// (there is no SIGTERM on Windows), so we instead kill the entire tree. +pub fn kill_process_tree(top_pid: &Pid) { + let mut system = System::new_all(); + system.refresh_processes(); + let processes = system.processes(); + + match processes.get(top_pid) { + None => return, + Some(top_process) => top_process.kill(), + }; + + kill_all_children(top_pid, processes); +} + +fn kill_all_children<'a>(top_pid: &'a Pid, processes: &'a HashMap) { + let mut pids_in_tree = HashSet::from([top_pid]); + + loop { + let current_tree_size = pids_in_tree.len(); + add_and_kill_direct_children(&mut pids_in_tree, processes); + if pids_in_tree.len() == current_tree_size { + break; + } + } +} + +fn add_and_kill_direct_children<'a>( + pids_in_tree: &mut HashSet<&'a Pid>, + processes: &'a HashMap, +) { + for (pid, parent_pid, process) in processes.iter().filter_map(|(pid, process)| { + process + .parent() + .map(|parent_pid| (pid, parent_pid, process)) + }) { + { + if pids_in_tree.contains(&parent_pid) { + pids_in_tree.insert(pid); + process.kill(); + } + } + } +}