From 77500a8f7c06e22d1b3b8b8c46a5068c072b08b7 Mon Sep 17 00:00:00 2001 From: Joerg Herbel Date: Tue, 28 Nov 2023 08:42:35 +0100 Subject: [PATCH] System test --- .github/workflows/system_test.yaml | 26 +++ .github/workflows/tests.yaml | 2 +- ci | 6 +- v2/robotmk/Cargo.lock | 91 +++++++++ v2/robotmk/Cargo.toml | 3 + v2/robotmk/src/config.rs | 22 +-- v2/robotmk/tests/minimal_suite/.gitignore | 1 + v2/robotmk/tests/minimal_suite/conda.yaml | 8 + v2/robotmk/tests/minimal_suite/lib/add.py | 10 + v2/robotmk/tests/minimal_suite/robot.yaml | 12 ++ v2/robotmk/tests/minimal_suite/tasks.robot | 21 +++ v2/robotmk/tests/test_scheduler.rs | 209 +++++++++++++++++++++ 12 files changed, 396 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/system_test.yaml create mode 100644 v2/robotmk/tests/minimal_suite/.gitignore create mode 100644 v2/robotmk/tests/minimal_suite/conda.yaml create mode 100644 v2/robotmk/tests/minimal_suite/lib/add.py create mode 100644 v2/robotmk/tests/minimal_suite/robot.yaml create mode 100644 v2/robotmk/tests/minimal_suite/tasks.robot create mode 100644 v2/robotmk/tests/test_scheduler.rs diff --git a/.github/workflows/system_test.yaml b/.github/workflows/system_test.yaml new file mode 100644 index 00000000..2664cedb --- /dev/null +++ b/.github/workflows/system_test.yaml @@ -0,0 +1,26 @@ +--- +name: "System tests" + +on: [push] + +jobs: + rcc: + uses: ./.github/workflows/rcc.yaml + + test_scheduler: + runs-on: windows-latest + needs: + - rcc + steps: + - uses: actions/download-artifact@v3 + with: + path: C:\ + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1.5.0 + with: + target: x86_64-pc-windows-gnu + - run: cargo test --target=x86_64-pc-windows-gnu --test test_scheduler -- --nocapture + working-directory: ${{ github.workspace }}/v2/robotmk/ + env: + TEST_DIR: C:\test_scheduler + RCC_BINARY_PATH: C:\artifact\windows64\rcc.exe diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index df69a981..19e0eaa7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ jobs: - run: cargo fmt -- --check working-directory: ${{ github.workspace }}/v2/robotmk/ - - run: cargo test --all-targets --target ${{ matrix.type.target }} + - run: cargo test --all-targets --target ${{ matrix.type.target }} -- --skip test_scheduler working-directory: ${{ github.workspace }}/v2/robotmk/ - run: cargo clippy --all-targets --target ${{ matrix.type.target }} -- --deny warnings diff --git a/ci b/ci index 632b8c16..3ecf3005 100755 --- a/ci +++ b/ci @@ -17,13 +17,13 @@ main() { cargo clippy --manifest-path "${cargo_toml_path}" --all-targets --target "${target}" -- --deny warnings ;; - 'cargo-test') - cargo test --manifest-path "${cargo_toml_path}" --all-targets --target "${target}" + 'cargo-test-unit') + cargo test --manifest-path "${cargo_toml_path}" --all-targets --target "${target}" -- --skip test_scheduler ;; 'check-all') exit_code=0 - for rust_step in fmt-check clippy test + for rust_step in fmt-check clippy test-unit do "${self}" "cargo-${rust_step}" exit_code=$(( exit_code + $? )) diff --git a/v2/robotmk/Cargo.lock b/v2/robotmk/Cargo.lock index 357de382..308fd88b 100644 --- a/v2/robotmk/Cargo.lock +++ b/v2/robotmk/Cargo.lock @@ -116,6 +116,21 @@ dependencies = [ "backtrace", ] +[[package]] +name = "assert_cmd" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -155,6 +170,17 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "bstr" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -309,6 +335,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "either" version = "1.9.0" @@ -474,6 +512,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -656,6 +703,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "predicates" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" +dependencies = [ + "anstyle", + "difflib", + "itertools", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro2" version = "1.0.69" @@ -737,6 +812,7 @@ name = "robotmk" version = "0.1.0" dependencies = [ "anyhow", + "assert_cmd", "base64", "camino", "chrono", @@ -905,6 +981,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "thiserror" version = "1.0.50" @@ -1010,6 +1092,15 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.4.0" diff --git a/v2/robotmk/Cargo.toml b/v2/robotmk/Cargo.toml index 72c798e3..8be929b1 100644 --- a/v2/robotmk/Cargo.toml +++ b/v2/robotmk/Cargo.toml @@ -24,6 +24,9 @@ tokio = { version = "1.33.0", features = ["full"] } tokio-util = { version = "0.7.10", features = ["full"] } walkdir = "2.4.0" +[dev-dependencies] +assert_cmd = "2.0.12" + [[bin]] name = "robotmk_agent" path = "src/bin/agent.rs" diff --git a/v2/robotmk/src/config.rs b/v2/robotmk/src/config.rs index d89e8e05..27e59965 100644 --- a/v2/robotmk/src/config.rs +++ b/v2/robotmk/src/config.rs @@ -1,7 +1,7 @@ use crate::section::Host; use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::from_str; use std::collections::HashMap; use std::fs::read_to_string; @@ -10,7 +10,7 @@ pub fn load(path: &Utf8Path) -> Result { Ok(from_str(&read_to_string(path)?)?) } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct Config { pub working_directory: Utf8PathBuf, pub results_directory: Utf8PathBuf, @@ -18,7 +18,7 @@ pub struct Config { pub suites: HashMap, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct SuiteConfig { pub robot_framework_config: RobotFrameworkConfig, pub execution_config: ExecutionConfig, @@ -28,13 +28,13 @@ pub struct SuiteConfig { pub host: Host, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct RobotFrameworkConfig { pub robot_target: Utf8PathBuf, pub command_line_args: Vec, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct ExecutionConfig { pub n_attempts_max: usize, pub retry_strategy: RetryStrategy, @@ -42,37 +42,37 @@ pub struct ExecutionConfig { pub timeout: u64, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum RetryStrategy { Incremental, Complete, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum EnvironmentConfig { System, Rcc(RCCEnvironmentConfig), } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct RCCEnvironmentConfig { pub robot_yaml_path: Utf8PathBuf, pub build_timeout: u64, pub env_json_path: Option, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum SessionConfig { Current, SpecificUser(UserSessionConfig), } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct UserSessionConfig { pub user_name: String, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub enum WorkingDirectoryCleanupConfig { MaxAgeSecs(u64), MaxExecutions(usize), diff --git a/v2/robotmk/tests/minimal_suite/.gitignore b/v2/robotmk/tests/minimal_suite/.gitignore new file mode 100644 index 00000000..3fec32c8 --- /dev/null +++ b/v2/robotmk/tests/minimal_suite/.gitignore @@ -0,0 +1 @@ +tmp/ diff --git a/v2/robotmk/tests/minimal_suite/conda.yaml b/v2/robotmk/tests/minimal_suite/conda.yaml new file mode 100644 index 00000000..0859ee83 --- /dev/null +++ b/v2/robotmk/tests/minimal_suite/conda.yaml @@ -0,0 +1,8 @@ +channels: + - conda-forge + +dependencies: + - python=3.11.6 + - pip=23.3.1 + - pip: + - rpaframework==27.7.0 diff --git a/v2/robotmk/tests/minimal_suite/lib/add.py b/v2/robotmk/tests/minimal_suite/lib/add.py new file mode 100644 index 00000000..c61a5f10 --- /dev/null +++ b/v2/robotmk/tests/minimal_suite/lib/add.py @@ -0,0 +1,10 @@ +def setup() -> None: + print("Setting up...") + + +def teardown() -> None: + print("Tearing down...") + + +def add(left: int, right: int) -> int: + return left + right diff --git a/v2/robotmk/tests/minimal_suite/robot.yaml b/v2/robotmk/tests/minimal_suite/robot.yaml new file mode 100644 index 00000000..6ee4ca61 --- /dev/null +++ b/v2/robotmk/tests/minimal_suite/robot.yaml @@ -0,0 +1,12 @@ +tasks: + execute: + command: + - python + - --version + +condaConfigFile: conda.yaml +artifactsDir: /tmp/outputdir # Leading slash is ignored, instead we get $(pwd)/tmp/outputdir/ +PATH: + - . +PYTHONPATH: + - . diff --git a/v2/robotmk/tests/minimal_suite/tasks.robot b/v2/robotmk/tests/minimal_suite/tasks.robot new file mode 100644 index 00000000..e9f09f3c --- /dev/null +++ b/v2/robotmk/tests/minimal_suite/tasks.robot @@ -0,0 +1,21 @@ +*** Settings *** +Documentation Test file for configuring RobotFramework + +Library ${CURDIR}/lib/add.py WITH NAME math + +Suite Setup math.setup +Suite Teardown math.teardown + + +*** Test Cases *** +Addition One + ${result}= math.add ${20} ${5} + Should Be Equal As Integers ${result} ${25} + +Addition Two + ${result}= math.add ${20} ${15} + Should Be Equal As Integers ${result} ${35} + +Addition Three + ${result}= math.add ${20} ${25} + Should Be Equal As Integers ${result} ${45} diff --git a/v2/robotmk/tests/test_scheduler.rs b/v2/robotmk/tests/test_scheduler.rs new file mode 100644 index 00000000..cc0c9608 --- /dev/null +++ b/v2/robotmk/tests/test_scheduler.rs @@ -0,0 +1,209 @@ +use anyhow::Result; +use assert_cmd::cargo::cargo_bin; +use camino::{Utf8Path, Utf8PathBuf}; +use robotmk::config::{ + Config, EnvironmentConfig, ExecutionConfig, RCCEnvironmentConfig, RetryStrategy, + RobotFrameworkConfig, SessionConfig, SuiteConfig, WorkingDirectoryCleanupConfig, +}; +use robotmk::section::Host; +use serde_json::to_string; +use std::env::var; +use std::ffi::OsStr; +use std::fs::{create_dir_all, remove_file, write}; +use std::path::Path; +use std::time::Duration; +use tokio::{process::Command, time::timeout}; +use walkdir::WalkDir; + +#[tokio::test(flavor = "multi_thread")] +async fn test_scheduler() -> Result<()> { + let test_dir = Utf8PathBuf::from(var("TEST_DIR")?); + create_dir_all(&test_dir)?; + let config = create_config( + &test_dir, + &Utf8PathBuf::from(var("CARGO_MANIFEST_DIR")?) + .join("tests") + .join("minimal_suite"), + var("RCC_BINARY_PATH")?, + ); + + run_scheduler(&test_dir, &config, 240).await?; + + assert_working_directory(&config.working_directory).await?; + assert_results_directory(&config.results_directory); + assert_rcc(&config.rcc_binary_path).await?; + + Ok(()) +} + +fn create_config( + test_dir: &Utf8Path, + suite_dir: &Utf8Path, + rcc_binary_path: impl Into, +) -> Config { + Config { + working_directory: test_dir.join("working"), + results_directory: test_dir.join("results"), + rcc_binary_path: rcc_binary_path.into(), + suites: [( + String::from("suite"), + SuiteConfig { + robot_framework_config: RobotFrameworkConfig { + robot_target: suite_dir.join("tasks.robot"), + command_line_args: vec![], + }, + execution_config: ExecutionConfig { + n_attempts_max: 1, + retry_strategy: RetryStrategy::Complete, + execution_interval_seconds: 20, + timeout: 10, + }, + environment_config: EnvironmentConfig::Rcc(RCCEnvironmentConfig { + robot_yaml_path: suite_dir.join("robot.yaml"), + build_timeout: 600, + env_json_path: None, + }), + session_config: SessionConfig::Current, + working_directory_cleanup_config: WorkingDirectoryCleanupConfig::MaxExecutions(4), + host: Host::Source, + }, + )] + .into(), + } +} + +async fn run_scheduler(test_dir: &Utf8Path, config: &Config, n_seconds_run: u64) -> Result<()> { + let config_path = test_dir.join("config.json"); + write(&config_path, to_string(&config)?)?; + let run_flag_path = test_dir.join("run_flag"); + write(&run_flag_path, "")?; + + let mut robotmk_cmd = Command::new(cargo_bin("robotmk")); + robotmk_cmd + .arg(config_path) + .arg("-vv") + .arg("--run-flag") + .arg(&run_flag_path); + let mut robotmk_child_proc = robotmk_cmd.spawn()?; + + assert!(timeout( + Duration::from_secs(n_seconds_run), + robotmk_child_proc.wait() + ) + .await + .is_err()); + remove_file(&run_flag_path)?; + assert!(timeout(Duration::from_secs(3), robotmk_child_proc.wait()) + .await + .is_ok()); + + Ok(()) +} + +async fn assert_working_directory(working_directory: &Utf8Path) -> Result<()> { + assert_working_directory_permissions(&working_directory).await?; + assert!(working_directory.is_dir()); + assert_eq!( + directory_entries(working_directory, 1), + ["environment_building_stdio", "rcc_setup", "suites"] + ); + assert_eq!( + directory_entries(working_directory.join("rcc_setup"), 1), + [ + "holotree_initialization_current_user.stderr", + "holotree_initialization_current_user.stdout", + "long_path_support_enabling.stderr", + "long_path_support_enabling.stdout", + "shared_holotree_init.stderr", + "shared_holotree_init.stdout", + "telemetry_disabling_current_user.stderr", + "telemetry_disabling_current_user.stdout" + ] + ); + assert_eq!( + directory_entries(working_directory.join("environment_building_stdio"), 1), + ["suite.stderr", "suite.stdout"] + ); + assert_eq!( + directory_entries(working_directory.join("suites"), 1), + ["suite"] + ); + assert!(!directory_entries(working_directory.join("suites").join("suite"), 1).is_empty(),); + Ok(()) +} + +async fn assert_working_directory_permissions(working_directory: &impl AsRef) -> Result<()> { + let mut icacls_command = Command::new("icacls.exe"); + icacls_command.arg(working_directory); + let stdout = String::from_utf8(icacls_command.output().await?.stdout)?; + assert!(stdout.contains("BUILTIN\\Users:(OI)(CI)(F)")); + Ok(()) +} + +fn assert_results_directory(results_directory: &Utf8Path) { + assert!(results_directory.is_dir()); + assert_eq!( + directory_entries(results_directory, 2), + [ + "environment_build_states.json", + "rcc_setup_failures.json", + "scheduler_phase.json", + "suites", + "suites\\suite.json" + ] + ); +} + +async fn assert_rcc(rcc_binary_path: impl AsRef) -> Result<()> { + assert_rcc_binary_permissions(&rcc_binary_path).await?; + assert_rcc_configuration(&rcc_binary_path).await?; + assert_rcc_longpath_support_enabled(&rcc_binary_path).await +} + +async fn assert_rcc_binary_permissions(rcc_binary_path: impl AsRef) -> Result<()> { + let mut icacls_command = Command::new("icacls.exe"); + icacls_command.arg(rcc_binary_path); + let stdout = String::from_utf8(icacls_command.output().await?.stdout)?; + assert!(stdout.contains("BUILTIN\\Users:(RX)")); + Ok(()) +} + +async fn assert_rcc_configuration(rcc_binary_path: impl AsRef) -> Result<()> { + let mut rcc_config_diag_command = Command::new(rcc_binary_path); + rcc_config_diag_command + .arg("configuration") + .arg("diagnostics"); + let stdout = String::from_utf8(rcc_config_diag_command.output().await?.stdout)?; + assert!(stdout.contains("telemetry-enabled ... \"false\"")); + assert!(stdout.contains("holotree-shared ... \"true\"")); + assert!(stdout.contains("holotree-global-shared ... \"true\"")); + Ok(()) +} + +async fn assert_rcc_longpath_support_enabled(rcc_binary_path: impl AsRef) -> Result<()> { + let mut rcc_config_diag_command = Command::new(rcc_binary_path); + rcc_config_diag_command + .arg("configuration") + .arg("longpaths"); + let stderr = String::from_utf8(rcc_config_diag_command.output().await?.stderr)?; + assert_eq!(stderr, "OK.\n"); + Ok(()) +} + +fn directory_entries(directory: impl AsRef, max_depth: usize) -> Vec { + WalkDir::new(&directory) + .max_depth(max_depth) + .into_iter() + .map(|dir_entry_result| { + dir_entry_result + .unwrap() + .path() + .strip_prefix(&directory) + .unwrap() + .to_str() + .unwrap() + .into() + }) + .filter(|entry: &String| !entry.is_empty()) + .collect() +}