From bb29ca0d1b6cde1db969cd1a781eeee2261bf4ea Mon Sep 17 00:00:00 2001 From: Christina Ying Wang Date: Wed, 13 Sep 2023 16:05:25 -0700 Subject: [PATCH] Implement config.json migration mechanism Signed-off-by: Christina Ying Wang --- src/join.rs | 27 +++-- src/main.rs | 1 + src/migrate.rs | 128 +++++++++++++++++++++++ tests/integration.rs | 237 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 387 insertions(+), 6 deletions(-) create mode 100644 src/migrate.rs diff --git a/src/join.rs b/src/join.rs index 8044232..e88fa6b 100644 --- a/src/join.rs +++ b/src/join.rs @@ -6,6 +6,7 @@ use crate::config_json::{ get_api_endpoint, get_root_certificate, merge_config_json, read_config_json, write_config_json, ConfigMap, }; +use crate::migrate::generate_config_json_migration; use crate::remote::{config_url, fetch_configuration, RemoteConfiguration}; use crate::schema::{read_os_config_schema, OsConfigSchema}; use crate::systemd; @@ -45,9 +46,12 @@ pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Resul !joining, )?; - let has_config_changes = has_config_changes(&schema, &remote_config)?; + let has_service_config_changes = has_service_config_changes(&schema, &remote_config)?; - if !has_config_changes { + let migrated_config_json = + generate_config_json_migration(&schema, &remote_config.config, &config_json.clone())?; + + if !has_service_config_changes && migrated_config_json == *config_json { info!("No configuration changes"); if !joining { @@ -66,8 +70,13 @@ pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Resul config_json, &schema, &remote_config, - has_config_changes, + has_service_config_changes, joining, + if migrated_config_json == *config_json { + None + } else { + Some(migrated_config_json) + }, ); if args.supervisor_exists { @@ -82,21 +91,27 @@ fn reconfigure_core( config_json: &ConfigMap, schema: &OsConfigSchema, remote_config: &RemoteConfiguration, - has_config_changes: bool, + has_service_config_changes: bool, joining: bool, + migrated_config_json: Option, ) -> Result<()> { if joining { write_config_json(&args.config_json_path, config_json)?; } - if has_config_changes { + if let Some(map) = migrated_config_json { + info!("Migrating config.json..."); + write_config_json(&args.config_json_path, &map)?; + } + + if has_service_config_changes { configure_services(schema, remote_config)?; } Ok(()) } -fn has_config_changes( +fn has_service_config_changes( schema: &OsConfigSchema, remote_config: &RemoteConfiguration, ) -> Result { diff --git a/src/main.rs b/src/main.rs index e2a7b61..3a83238 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,7 @@ mod generate; mod join; mod leave; mod logger; +mod migrate; mod random; mod remote; mod schema; diff --git a/src/migrate.rs b/src/migrate.rs new file mode 100644 index 0000000..89da032 --- /dev/null +++ b/src/migrate.rs @@ -0,0 +1,128 @@ +// Config.json migration module +// +// Provides methods for migrating config.json fields based on remote directives +// from /os/vX/config. Limits migrated fields based on os-config.json schema +// whitelist. + +use crate::config_json::ConfigMap; +use crate::remote::ConfigMigrationInstructions; +use crate::schema::OsConfigSchema; +use anyhow::Result; +use std::collections::HashMap; + +pub fn generate_config_json_migration( + schema: &OsConfigSchema, + migration_config: &ConfigMigrationInstructions, + config_json: &ConfigMap, +) -> Result { + info!("Checking for config.json migrations..."); + + let mut new_config = config_json.clone(); + + handle_update_directives( + schema, + &migration_config.overrides, + config_json, + &mut new_config, + ); + + Ok(new_config) +} + +fn handle_update_directives( + schema: &OsConfigSchema, + to_update: &HashMap, + config_json: &ConfigMap, + new_config: &mut ConfigMap, +) { + for key in to_update.keys() { + if !schema.config.whitelist.contains(key) { + debug!("Key `{}` not in whitelist, skipping", key); + continue; + } + + if let Some(future) = to_update.get(key) { + if !config_json.contains_key(key) { + info!("Key `{}` not found, will insert", key); + new_config.insert(key.to_string(), future.clone()); + } else if let Some(current) = config_json.get(key) { + if current != future { + info!( + "Key `{}` found with current value `{}`, will update to `{}`", + key, current, future + ); + new_config.insert(key.to_string(), future.clone()); + } else { + debug!( + "Key `{}` found with current value `{}` equal to update value `{}`, skipping", + key, current, future + ); + } + } + } + } +} + +mod tests { + #[test] + fn test_generate_config_json_migration() { + let config_json = r#" + { + "deadbeef": 1, + "deadca1f": "2", + "deadca2f": true, + "deadca3f": "string1" + } + "# + .to_string(); + + let schema = r#" + { + "services": [ + ], + "keys": [], + "config": { + "whitelist": [ + "deadbeef", + "deadca1f", + "deadca2f", + "deadca3f", + "deadca4f" + ] + } + } + "# + .to_string(); + + let configuration = unindent::unindent( + r#" + { + "overrides": { + "deadbeef": 2, + "deadca1f": "3", + "deadca2f": false, + "deadca3f": "string0", + "deadca4f": "new_field", + "not_on_whitelist1": "not_on_whitelist" + } + } + "#, + ); + + let old_config = serde_json::from_str::(&config_json).unwrap(); + + let new_config = super::generate_config_json_migration( + &serde_json::from_str(&schema).unwrap(), + &serde_json::from_str(&configuration).unwrap(), + &old_config, + ) + .unwrap(); + + assert_eq!(new_config.get("deadbeef").unwrap(), 2); + assert_eq!(new_config.get("deadca1f").unwrap(), "3"); + assert_eq!(new_config.get("deadca2f").unwrap(), false); + assert_eq!(new_config.get("deadca3f").unwrap(), "string0"); + assert_eq!(new_config.get("deadca4f").unwrap(), "new_field"); + assert!(new_config.get("not_on_whitelist1").is_none()); + } +} diff --git a/tests/integration.rs b/tests/integration.rs index d0e5022..08c9ca4 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -139,6 +139,7 @@ fn join() { r#" Fetching service configuration from http://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... Writing {tmp_dir_path}/config.json @@ -316,6 +317,7 @@ fn join_flasher() { r#" Fetching service configuration from http://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... Writing {tmp_dir_path}/config.json @@ -453,6 +455,7 @@ fn join_with_root_certificate() { r#" Fetching service configuration from https://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... @@ -674,6 +677,7 @@ fn reconfigure() { r#" Fetching service configuration from http://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... @@ -821,6 +825,7 @@ fn reconfigure_stored() { r#" Fetching service configuration from http://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... @@ -998,6 +1003,7 @@ fn update() { r#" Fetching service configuration from http://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... {tmp_dir_path}/not-a-service-1.conf updated @@ -1163,6 +1169,7 @@ fn update_no_config_changes() { r#" Fetching service configuration from http://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes "#, )); @@ -1267,6 +1274,7 @@ fn update_with_root_certificate() { r#" Fetching service configuration from https://localhost:{port}/os/v1/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes "#, )); @@ -1818,6 +1826,235 @@ fn generate_api_key_new() { ); } +#[test] +#[timeout(10000)] +fn migrate_config_json() { + let port = 31014; + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path().to_str().unwrap().to_string(); + + let config_json = format!( + r#" + {{ + "deviceApiKey": "abcdef", + "deviceType": "intel-nuc", + "hostname": "balena", + "persistentLogging": false, + "applicationName": "aaaaaa", + "applicationId": 1234567, + "userId": 654321, + "appUpdatePollInterval": 900000, + "listenPort": 48484, + "vpnPort": 443, + "apiEndpoint": "http://{}", + "vpnEndpoint": "vpn.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta.balenadev.io", + "apiKey": "12345678abcd1234efgh1234567890ab", + "version": "9.99.9+rev1", + "deadbeef": "12345678", + "mixpanelToken": "12345678abcd1234efgh1234567890ab" + }} + "#, + server_address(port) + ); + + let config_json_path = create_tmp_file(&tmp_dir, "config.json", &config_json, None); + + let schema = r#" + { + "services": [ + ], + "keys": ["apiKey", "apiEndpoint", "vpnEndpoint"], + "config": { + "whitelist": [ + "logsEndpoint", + "mixpanelToken", + "registryEndpoint", + "deltaEndpoint", + "applicationId", + "persistentLogging", + "deadbeef" + ] + } + } + "# + .to_string(); + + let os_config_path = create_tmp_file(&tmp_dir, "os-config.json", &schema, None); + + // - apiEndpoint: not on whitelist, should skip + // - logsEndpoint: value not present, should insert + // - registryEndpoint: value unchanged, should skip + // - deltaEndpoint: value changed, should update (String) + // - applicationId: value changed, should update (u64) + // - persistentLogging: value changed, should update (bool) + // - deadbeef: value changed, should update (stringified u64) + let configuration = unindent::unindent( + r#" + { + "services": {}, + "config": { + "overrides": { + "apiEndpoint": "http://api.balenadev.io", + "logsEndpoint": "https://logs.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta2.balenadev.io", + "applicationId": 1234568, + "persistentLogging": true, + "deadbeef": "1234567890" + } + } + } + "#, + ); + + let mut serve = serve_config(configuration, false, port); + + // This is unfortunately necessary because the output for each line that + // begins with `Key` may vary in order. + let output_patterns = [ + format!("Fetching service configuration from http://localhost:{}/os/v1/config...\nService configuration retrieved\nChecking for config.json migrations...", port), + "Key `logsEndpoint` not found, will insert".into(), + "Key `deltaEndpoint` found with current value `\"https://delta.balenadev.io\"`, will update to `\"https://delta2.balenadev.io\"`".into(), + "Key `deadbeef` found with current value `\"12345678\"`, will update to `\"1234567890\"`".into(), + "Key `persistentLogging` found with current value `false`, will update to `true`".into(), + "Key `applicationId` found with current value `1234567`, will update to `1234568`".into(), + format!("Stopping balena-supervisor.service...\nAwaiting balena-supervisor.service to exit...\nMigrating config.json...\nWriting {}/config.json\nStarting balena-supervisor.service...", tmp_dir_path), + ]; + + let output = get_base_command() + .args(["update"]) + .timeout(Duration::from_secs(5)) + .envs(os_config_env(&os_config_path, &config_json_path)) + .assert() + .success() + .get_output() + .to_owned(); + + let string_output = String::from_utf8(output.stdout).unwrap(); + + for pattern in output_patterns.iter() { + // write string_output to local file for debugging + assert!(string_output.contains(pattern)); + } + + validate_json_file( + &config_json_path, + &format!( + r#" + {{ + "deviceApiKey": "abcdef", + "deviceType": "intel-nuc", + "hostname": "balena", + "persistentLogging": true, + "applicationName": "aaaaaa", + "applicationId": 1234568, + "userId": 654321, + "appUpdatePollInterval": 900000, + "listenPort": 48484, + "vpnPort": 443, + "apiEndpoint": "http://{}", + "vpnEndpoint": "vpn.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta2.balenadev.io", + "apiKey": "12345678abcd1234efgh1234567890ab", + "version": "9.99.9+rev1", + "deadbeef": "1234567890", + "mixpanelToken": "12345678abcd1234efgh1234567890ab", + "logsEndpoint": "https://logs.balenadev.io" + }} + "#, + server_address(port) + ), + false, + ); + + serve.stop(); +} + +#[test] +#[timeout(10000)] +fn ignore_unknown_cloud_config_fields() { + let port = 31015; + let tmp_dir = TempDir::new().unwrap(); + + let config_json = format!( + r#" + {{ + "deviceApiKey": "abcdef", + "deviceType": "intel-nuc", + "hostname": "balena", + "persistentLogging": false, + "applicationName": "aaaaaa", + "applicationId": 1234567, + "userId": 654321, + "appUpdatePollInterval": 900000, + "listenPort": 48484, + "vpnPort": 443, + "apiEndpoint": "http://{}", + "vpnEndpoint": "vpn.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta.balenadev.io", + "apiKey": "12345678abcd1234efgh1234567890ab", + "version": "9.99.9+rev1", + "mixpanelToken": "12345678abcd1234efgh1234567890ab" + }} + "#, + server_address(port) + ); + + let config_json_path = create_tmp_file(&tmp_dir, "config.json", &config_json, None); + + let schema = r#" + { + "services": [ + ], + "keys": ["apiKey", "apiEndpoint", "vpnEndpoint"], + "config": { + "whitelist": [] + } + } + "# + .to_string(); + + let os_config_path = create_tmp_file(&tmp_dir, "os-config.json", &schema, None); + + let configuration = unindent::unindent( + r#" + { + "services": {}, + "config": { + "overrides": {} + }, + "unknown_field": "unknown_value" + } + "#, + ); + + let mut serve = serve_config(configuration, false, port); + + let output = unindent::unindent(&format!( + r#" + Fetching service configuration from http://localhost:{port}/os/v1/config... + Service configuration retrieved + Checking for config.json migrations... + No configuration changes + "#, + )); + + get_base_command() + .args(["update"]) + .timeout(Duration::from_secs(5)) + .envs(os_config_env(&os_config_path, &config_json_path)) + .assert() + .success() + .stdout(output); + + validate_json_file(&config_json_path, &config_json, false); + + serve.stop(); +} /******************************************************************************* * os-config launch */