From 6a99fc73d9fda3a72e7f63402ee969de80aaf84e Mon Sep 17 00:00:00 2001 From: Joerg Herbel Date: Mon, 18 Sep 2023 12:02:49 +0200 Subject: [PATCH] Rust scheduler: Implement setup step upon startup CMK-14522 --- v2/rust/Cargo.lock | 83 ++++++++++++++++++++++++++- v2/rust/Cargo.toml | 1 + v2/rust/src/environment.rs | 5 ++ v2/rust/src/main.rs | 14 ++++- v2/rust/src/results.rs | 9 +++ v2/rust/src/setup.rs | 112 +++++++++++++++++++++++++++++++++++++ 6 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 v2/rust/src/environment.rs create mode 100644 v2/rust/src/results.rs create mode 100644 v2/rust/src/setup.rs diff --git a/v2/rust/Cargo.lock b/v2/rust/Cargo.lock index e196d588..1eeed02a 100644 --- a/v2/rust/Cargo.lock +++ b/v2/rust/Cargo.lock @@ -98,6 +98,17 @@ dependencies = [ "backtrace", ] +[[package]] +name = "atomicwrites" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1163d9d7c51de51a2b79d6df5e8888d11e9df17c752ce4a285fb6ca1580734e" +dependencies = [ + "rustix 0.37.23", + "tempfile", + "windows-sys", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -119,6 +130,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.4.0" @@ -233,6 +250,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "flexi_logger" version = "0.26.1" @@ -296,6 +319,17 @@ dependencies = [ "cc", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -303,7 +337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix", + "rustix 0.38.13", "windows-sys", ] @@ -334,6 +368,12 @@ version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "linux-raw-sys" version = "0.4.7" @@ -422,6 +462,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.9.5" @@ -456,6 +505,7 @@ name = "robotmk" version = "0.1.0" dependencies = [ "anyhow", + "atomicwrites", "chrono", "clap", "flexi_logger", @@ -471,16 +521,30 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.37.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys", +] + [[package]] name = "rustix" version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ - "bitflags", + "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.7", "windows-sys", ] @@ -538,6 +602,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix 0.38.13", + "windows-sys", +] + [[package]] name = "thiserror" version = "1.0.48" diff --git a/v2/rust/Cargo.toml b/v2/rust/Cargo.toml index 1b03b7a6..9ca28245 100644 --- a/v2/rust/Cargo.toml +++ b/v2/rust/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] anyhow = { version = "*", features = ["backtrace"] } +atomicwrites = "*" chrono = "0.4.31" clap = { version = "*", features = ["derive"] } flexi_logger = "*" diff --git a/v2/rust/src/environment.rs b/v2/rust/src/environment.rs new file mode 100644 index 00000000..1760efe5 --- /dev/null +++ b/v2/rust/src/environment.rs @@ -0,0 +1,5 @@ +use std::path::{Path, PathBuf}; + +pub fn environment_building_stdio_directory(working_directory: &Path) -> PathBuf { + working_directory.join("environment_building_stdio") +} diff --git a/v2/rust/src/main.rs b/v2/rust/src/main.rs index 0bf55a55..525514d6 100644 --- a/v2/rust/src/main.rs +++ b/v2/rust/src/main.rs @@ -2,23 +2,31 @@ pub mod attempt; mod cli; mod config; +mod environment; mod logging; pub mod parse_xml; +mod results; +mod setup; -use anyhow::Context; +use anyhow::{Context, Result}; use clap::Parser; use log::{debug, info}; use logging::log_and_return_error; -fn main() -> anyhow::Result<()> { +fn main() -> Result<()> { let args = cli::Args::parse(); logging::init(args.log_specification(), &args.log_path)?; info!("Program started and logging set up"); - let _config = config::load(&args.config_path) + let conf = config::load(&args.config_path) .context("Configuration loading failed") .map_err(log_and_return_error)?; debug!("Configuration loaded"); + setup::setup(&conf) + .context("Setup failed") + .map_err(log_and_return_error)?; + debug!("Setup completed"); + Ok(()) } diff --git a/v2/rust/src/results.rs b/v2/rust/src/results.rs new file mode 100644 index 00000000..8c769e74 --- /dev/null +++ b/v2/rust/src/results.rs @@ -0,0 +1,9 @@ +use std::path::{Path, PathBuf}; + +pub fn suite_results_directory(results_directory: &Path) -> PathBuf { + results_directory.join("suites") +} + +pub fn suite_result_file(suite_results_dir: &Path, suite_name: &str) -> PathBuf { + suite_results_dir.join(format!("{}.json", suite_name)) +} diff --git a/v2/rust/src/setup.rs b/v2/rust/src/setup.rs new file mode 100644 index 00000000..f401e770 --- /dev/null +++ b/v2/rust/src/setup.rs @@ -0,0 +1,112 @@ +use anyhow::{Context, Result}; +use std::collections::HashSet; +use std::fs::create_dir_all; +use std::path::{Path, PathBuf}; + +use super::config::Config; +use super::environment::environment_building_stdio_directory; +use super::results::{suite_result_file, suite_results_directory}; + +pub fn setup(config: &Config) -> Result<()> { + create_dir_all(&config.working_directory).context("Failed to create working directory")?; + create_dir_all(environment_building_stdio_directory( + &config.working_directory, + )) + .context("Failed to create environment building stdio directory")?; + create_dir_all(&config.results_directory).context("Failed to create results directory")?; + create_dir_all(suite_results_directory(&config.results_directory)) + .context("Failed to create suite results directory")?; + clean_up_results_directory_atomic(config) +} + +fn clean_up_results_directory_atomic(config: &Config) -> Result<()> { + let suite_results_directory = suite_results_directory(&config.results_directory); + let result_files_to_keep: HashSet = HashSet::from_iter(result_files_to_keep( + &suite_results_directory, + config + .suites() + .into_iter() + .map(|(suite_name, _suite_config)| suite_name.as_str()), + )); + let currently_present_result_files = + HashSet::from_iter(currently_present_result_files(&suite_results_directory)?); + let result_files_to_remove = currently_present_result_files.difference(&result_files_to_keep); + remove_files_atomic( + &suite_results_directory.join("deprecated_result"), + result_files_to_remove.into_iter(), + ) +} + +fn result_files_to_keep<'a>( + suite_results_directory: &std::path::Path, + suite_names: impl Iterator, +) -> Vec { + let mut wanted_files_in_results_dir = vec![]; + + for suite_name in suite_names { + wanted_files_in_results_dir.push(suite_result_file(suite_results_directory, suite_name)); + } + + wanted_files_in_results_dir +} + +fn currently_present_result_files( + suite_results_directory: &std::path::Path, +) -> Result> { + let mut result_files = vec![]; + + for dir_entry in std::fs::read_dir(suite_results_directory).context(format!( + "Failed to read entries of results directory {}", + suite_results_directory.display() + ))? { + let dir_entry = dir_entry.context(format!( + "Failed to read entries of results directory {}", + suite_results_directory.display() + ))?; + if dir_entry + .file_type() + .context(format!( + "Failed to determine file type of {}", + dir_entry.path().display() + ))? + .is_file() + { + result_files.push(dir_entry.path()) + } + } + + Ok(result_files) +} + +fn remove_files_atomic<'a>( + intermediate_path_for_move: &Path, + files_to_be_removed: impl Iterator, +) -> Result<()> { + for path in files_to_be_removed { + atomicwrites::replace_atomic(path, intermediate_path_for_move).context(format!( + "Failed to move {} to {}", + path.display(), + intermediate_path_for_move.display() + ))?; + } + + let _ = std::fs::remove_file(intermediate_path_for_move); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_result_files_to_keep() { + assert_eq!( + result_files_to_keep(&PathBuf::from("/a/b/c"), ["suite1", "suite2"].into_iter()), + vec![ + PathBuf::from("/a/b/c/suite1.json"), + PathBuf::from("/a/b/c/suite2.json") + ] + ) + } +}