From 06bcbb538ab14ff67608dd0b0a0bd12a91a92178 Mon Sep 17 00:00:00 2001 From: Joerg Herbel Date: Mon, 18 Sep 2023 16:29:42 +0200 Subject: [PATCH] Rust scheduler: environment building CMK-14557 --- v2/rust/src/environment.rs | 229 +++++++++++++++++++++++++++++++++++++ v2/rust/src/main.rs | 17 +-- v2/rust/src/results.rs | 71 ++++++++++++ 3 files changed, 306 insertions(+), 11 deletions(-) diff --git a/v2/rust/src/environment.rs b/v2/rust/src/environment.rs index 1760efe5..3dab3db4 100644 --- a/v2/rust/src/environment.rs +++ b/v2/rust/src/environment.rs @@ -1,5 +1,234 @@ +use super::child_process_supervisor::{ChildProcessOutcome, ChildProcessSupervisor}; +use super::config::{Config, EnvironmentConfig}; +use super::results::{EnvironmentBuildStatesAdministrator, EnvironmentBuildStatus}; +use super::termination::TerminationFlag; +use anyhow::{Context, Result}; +use log::{debug, error, info}; use std::path::{Path, PathBuf}; +use std::process::Command; pub fn environment_building_stdio_directory(working_directory: &Path) -> PathBuf { working_directory.join("environment_building_stdio") } + +pub fn build_environments(config: &Config, termination_flag: &TerminationFlag) -> Result<()> { + let suites = config.suites(); + let mut environment_build_states_administrator = + EnvironmentBuildStatesAdministrator::new_with_pending( + suites + .iter() + .map(|(suite_name, _suite_config)| suite_name.to_owned()), + &config.working_directory, + &config.results_directory, + ); + environment_build_states_administrator.write_atomic()?; + let env_building_stdio_directory = + environment_building_stdio_directory(&config.working_directory); + + for (suite_name, suite_config) in suites { + match Environment::new(suite_name, &suite_config.environment_config).build_instructions() { + Some(mut build_instructions) => { + info!("Building environment for suite {}", suite_name); + environment_build_states_administrator + .insert_and_write_atomic(suite_name, EnvironmentBuildStatus::InProgress)?; + configure_stdio_of_environment_build( + &env_building_stdio_directory, + suite_name, + &mut build_instructions.command, + ) + .context("Configuring stdio of environment build process failed")?; + environment_build_states_administrator.insert_and_write_atomic( + suite_name, + run_environment_build(ChildProcessSupervisor { + command: build_instructions.command, + timeout: build_instructions.timeout, + termination_flag, + })?, + )?; + } + None => { + debug!("Nothing to do for suite {}", suite_name); + environment_build_states_administrator + .insert_and_write_atomic(suite_name, EnvironmentBuildStatus::NotNeeded)?; + } + } + } + + Ok(()) +} + +fn configure_stdio_of_environment_build( + stdio_directory: &Path, + suite_name: &str, + build_command: &mut Command, +) -> Result<()> { + let path_stdout = stdio_directory.join(format!("{}.stdout", suite_name)); + let path_stderr = stdio_directory.join(format!("{}.stderr", suite_name)); + build_command + .stdout(std::fs::File::create(&path_stdout).context(format!( + "Failed to open {} for stdout capturing", + &path_stdout.display() + ))?) + .stderr(std::fs::File::create(&path_stderr).context(format!( + "Failed to open {} for stderr capturing", + &path_stderr.display() + ))?); + Ok(()) +} + +fn run_environment_build( + build_process_supervisor: ChildProcessSupervisor, +) -> Result { + match build_process_supervisor + .run() + .context("Environment building failed")? + { + ChildProcessOutcome::Exited(exit_status) => { + if exit_status.success() { + debug!("Environmenent building succeeded"); + Ok(EnvironmentBuildStatus::Success) + } else { + error!("Environment building not sucessful, suite will most likely not execute"); + Ok(EnvironmentBuildStatus::Failure) + } + } + ChildProcessOutcome::TimedOut => { + error!("Environment building timed out, suite will most likely not execute"); + Ok(EnvironmentBuildStatus::Timeout) + } + } +} + +pub enum Environment { + System(SystemEnvironment), + Rcc(RCCEnvironment), +} + +pub struct SystemEnvironment {} + +pub struct RCCEnvironment { + binary_path: PathBuf, + robocorp_home_path: PathBuf, + robot_yaml_path: PathBuf, + controller: String, + space: String, + build_timeout: u64, +} + +impl Environment { + pub fn new(suite_name: &str, environment_config: &EnvironmentConfig) -> Self { + match environment_config { + EnvironmentConfig::System => Self::System(SystemEnvironment {}), + EnvironmentConfig::Rcc(rcc_environment_config) => Self::Rcc(RCCEnvironment { + binary_path: rcc_environment_config.binary_path.clone(), + robocorp_home_path: rcc_environment_config.robocorp_home_path.clone(), + robot_yaml_path: rcc_environment_config.robot_yaml_path.clone(), + controller: String::from("robotmk"), + space: suite_name.to_string(), + build_timeout: rcc_environment_config.build_timeout, + }), + } + } + + fn build_instructions(&self) -> Option { + match self { + Self::System(system_environment) => system_environment.build_command(), + Self::Rcc(rcc_environment) => rcc_environment.build_command(), + } + } +} + +struct BuildInstructions { + command: Command, + timeout: u64, +} + +impl SystemEnvironment { + fn build_command(&self) -> Option { + None + } +} + +impl RCCEnvironment { + fn build_command(&self) -> Option { + let mut build_cmd = Command::new(&self.binary_path); + self.apply_current_settings(build_cmd.arg("holotree").arg("variables").arg("--json")); + Some(BuildInstructions { + command: build_cmd, + timeout: self.build_timeout, + }) + } + + fn apply_current_settings<'a>(&self, command: &'a mut Command) -> &'a mut Command { + command + .env("ROBOCORP_HOME", &self.robocorp_home_path) + .arg("--robot") + .arg(&self.robot_yaml_path) + .arg("--controller") + .arg(&self.controller) + .arg("--space") + .arg(&self.space) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::RCCEnvironmentConfig; + + #[test] + fn environment_from_system_config() { + assert!(Environment::new("my_suite", &EnvironmentConfig::System) + .build_instructions() + .is_none()) + } + + #[test] + fn environment_from_rcc_config() { + assert!(Environment::new( + "my_suite", + &EnvironmentConfig::Rcc(RCCEnvironmentConfig { + binary_path: PathBuf::from("/bin/rcc"), + robocorp_home_path: PathBuf::from("/robocorp_home"), + robot_yaml_path: PathBuf::from("/a/b/c/robot.yaml"), + build_timeout: 60, + }) + ) + .build_instructions() + .is_some()) + } + + #[test] + fn rcc_build_command() { + let mut expected = Command::new("/bin/rcc"); + expected + .arg("holotree") + .arg("variables") + .arg("--json") + .arg("--robot") + .arg("/a/b/c/robot.yaml") + .arg("--controller") + .arg("robotmk") + .arg("--space") + .arg("my_suite") + .env("ROBOCORP_HOME", "/robocorp_home"); + + assert_eq!( + format!( + "{:?}", + RCCEnvironment { + binary_path: PathBuf::from("/bin/rcc"), + robocorp_home_path: PathBuf::from("/robocorp_home"), + robot_yaml_path: PathBuf::from("/a/b/c/robot.yaml"), + controller: String::from("robotmk"), + space: String::from("my_suite"), + build_timeout: 123, + } + .build_command() + .unwrap() + .command, + ), + format!("{:?}", expected) + ) + } +} diff --git a/v2/rust/src/main.rs b/v2/rust/src/main.rs index c964456e..7e4e370d 100644 --- a/v2/rust/src/main.rs +++ b/v2/rust/src/main.rs @@ -12,11 +12,8 @@ mod termination; use anyhow::{Context, Result}; use clap::Parser; -use log::{debug, info, warn}; +use log::{debug, info}; use logging::log_and_return_error; -use std::process::exit; -use std::thread::sleep; -use std::time::Duration; fn main() -> Result<()> { let args = cli::Args::parse(); @@ -38,11 +35,9 @@ fn main() -> Result<()> { .map_err(log_and_return_error)?; debug!("Termination control set up"); - loop { - if termination_flag.should_terminate() { - warn!("Termination signal received, shutting down"); - exit(1); - } - sleep(Duration::from_millis(100)) - } + info!("Starting environment building"); + environment::build_environments(&conf, &termination_flag).map_err(log_and_return_error)?; + info!("Environment building finished"); + + Ok(()) } diff --git a/v2/rust/src/results.rs b/v2/rust/src/results.rs index 8c769e74..6b5fe279 100644 --- a/v2/rust/src/results.rs +++ b/v2/rust/src/results.rs @@ -1,4 +1,9 @@ +use anyhow::{Context, Result}; +use atomicwrites::{AtomicFile, OverwriteBehavior}; +use serde::Serialize; +use serde_json::to_string; use std::path::{Path, PathBuf}; +use std::{collections::HashMap, io::Write}; pub fn suite_results_directory(results_directory: &Path) -> PathBuf { results_directory.join("suites") @@ -7,3 +12,69 @@ pub fn suite_results_directory(results_directory: &Path) -> PathBuf { pub fn suite_result_file(suite_results_dir: &Path, suite_name: &str) -> PathBuf { suite_results_dir.join(format!("{}.json", suite_name)) } + +pub fn write_file_atomic(content: &str, working_directory: &Path, final_path: &Path) -> Result<()> { + AtomicFile::new_with_tmpdir( + final_path, + OverwriteBehavior::AllowOverwrite, + working_directory, + ) + .write(|f| f.write_all(content.as_bytes())) + .context(format!( + "Atomic write failed. Working directory: {}, final path: {}.", + working_directory.display(), + final_path.display() + )) +} + +pub struct EnvironmentBuildStatesAdministrator<'a> { + build_states: HashMap<&'a String, EnvironmentBuildStatus>, + working_directory: &'a Path, + results_directory: &'a Path, +} + +impl<'a> EnvironmentBuildStatesAdministrator<'a> { + pub fn new_with_pending( + suite_names: impl Iterator, + working_directory: &'a Path, + results_directory: &'a Path, + ) -> EnvironmentBuildStatesAdministrator<'a> { + Self { + build_states: HashMap::from_iter( + suite_names.map(|suite_name| (suite_name, EnvironmentBuildStatus::Pending)), + ), + working_directory, + results_directory, + } + } + + pub fn write_atomic(&self) -> Result<()> { + write_file_atomic( + &to_string(&self.build_states) + .context("Serializing environment build states failed")?, + self.working_directory, + &self.results_directory.join("environment_build_states.json"), + ) + .context("Writing environment build states failed") + } + + pub fn insert_and_write_atomic( + &mut self, + suite_name: &'a String, + environment_build_status: EnvironmentBuildStatus, + ) -> Result<()> { + self.build_states + .insert(suite_name, environment_build_status); + self.write_atomic() + } +} + +#[derive(Serialize)] +pub enum EnvironmentBuildStatus { + Success, + Failure, + Timeout, + NotNeeded, + Pending, + InProgress, +}