From 4258fa4898bee2e073ccf0f62692d63ce2e07650 Mon Sep 17 00:00:00 2001 From: Christina Ying Wang Date: Wed, 23 Aug 2023 19:42:01 -0700 Subject: [PATCH] Implement config.json migration mechanism Signed-off-by: Christina Ying Wang --- src/join.rs | 27 +++++-- src/main.rs | 1 + src/migrate.rs | 141 +++++++++++++++++++++++++++++++++++++ tests/integration.rs | 164 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 327 insertions(+), 6 deletions(-) create mode 100644 src/migrate.rs diff --git a/src/join.rs b/src/join.rs index 1088d41..fff529e 100644 --- a/src/join.rs +++ b/src/join.rs @@ -6,6 +6,7 @@ use crate::config_json::{ get_api_endpoint, get_root_certificate, merge_config_json, read_config_json, write_config_json, ConfigMap, }; +use crate::migrate::generate_config_json_migration; use crate::remote::{config_url, fetch_configuration, RemoteConfiguration}; use crate::schema::{read_os_config_schema, OsConfigSchema}; use crate::systemd; @@ -45,9 +46,12 @@ pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Resul !joining, )?; - let has_config_changes = has_config_changes(&schema, &remote_config)?; + let has_service_config_changes = has_service_config_changes(&schema, &remote_config)?; - if !has_config_changes { + let migrated_config_json = + generate_config_json_migration(&schema, &remote_config.config, &config_json.clone())?; + + if !has_service_config_changes && migrated_config_json == *config_json { info!("No configuration changes"); if !joining { @@ -66,8 +70,13 @@ pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Resul config_json, &schema, &remote_config, - has_config_changes, + has_service_config_changes, joining, + if migrated_config_json == *config_json { + None + } else { + Some(migrated_config_json) + }, ); if args.supervisor_exists { @@ -82,21 +91,27 @@ fn reconfigure_core( config_json: &ConfigMap, schema: &OsConfigSchema, remote_config: &RemoteConfiguration, - has_config_changes: bool, + has_service_config_changes: bool, joining: bool, + migrated_config_json: Option, ) -> Result<()> { if joining { write_config_json(&args.config_json_path, config_json)?; } - if has_config_changes { + if migrated_config_json.is_some() { + info!("Migrating config.json..."); + write_config_json(&args.config_json_path, &migrated_config_json.unwrap())?; + } + + if has_service_config_changes { configure_services(schema, remote_config)?; } Ok(()) } -fn has_config_changes( +fn has_service_config_changes( schema: &OsConfigSchema, remote_config: &RemoteConfiguration, ) -> Result { diff --git a/src/main.rs b/src/main.rs index e2a7b61..3a83238 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,7 @@ mod generate; mod join; mod leave; mod logger; +mod migrate; mod random; mod remote; mod schema; diff --git a/src/migrate.rs b/src/migrate.rs new file mode 100644 index 0000000..09c290b --- /dev/null +++ b/src/migrate.rs @@ -0,0 +1,141 @@ +// Config.json migration module +// +// Provides methods for migrating config.json fields based on remote directives +// from /os/vX/config. Limits migrated fields based on os-config.json schema +// whitelist. + +use crate::config_json::ConfigMap; +use crate::remote::ConfigMigrationInstructions; +use crate::schema::OsConfigSchema; +use anyhow::Result; + +pub fn generate_config_json_migration( + schema: &OsConfigSchema, + migration_config: &ConfigMigrationInstructions, + config_json: &ConfigMap, +) -> Result { + info!("Checking for config.json migrations..."); + + let mut new_config = config_json + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::(); + + let keys_to_update = migration_config.update.keys().cloned().collect::>(); + + for key in keys_to_update.iter().chain(migration_config.delete.iter()) { + if !schema.config.whitelist.contains(key) { + info!("Key `{}` not in whitelist, skipping", key); + continue; + } + + // Key is marked for deletion + if migration_config.delete.contains(key) { + if config_json.contains_key(key) { + info!("Key `{}` found, will delete", key); + new_config.remove(key); + } else { + info!("Key `{}` not found for deletion, skipping", key); + } + } else if migration_config.update.contains_key(key) { + // Key is marked for update + // If key is in both delete & update, delete takes precedence. + if let Some(future) = migration_config.update.get(key) { + if !config_json.contains_key(key) { + info!("Key `{}` not found, will insert", key); + new_config.insert(key.to_string(), future.clone()); + } else if let Some(current) = config_json.get(key) { + if current != future { + info!( + "Key `{}` found with current value `{}`, will update to `{}`", + key, current, future + ); + new_config.insert(key.to_string(), future.clone()); + } else { + info!( + "Key `{}` found with current value `{}` equal to update value `{}`, skipping", + key, current, future + ); + } + } + } + } + } + + Ok(new_config) +} + +mod tests { + #[test] + fn test_generate_config_json_migration() { + let config_json = format!( + r#" + {{ + "deadbeef": 1, + "deadca1f": "2", + "deadca2f": true, + "deadca3f": "string1", + "deadca5f": "to_delete", + "on_both_lists": "on_both_lists" + }} + "# + ); + + let schema = format!( + r#" + {{ + "services": [ + ], + "config": {{ + "whitelist": [ + "deadbeef", + "deadca1f", + "deadca2f", + "deadca3f", + "deadca4f", + "deadca5f", + "on_both_lists" + ], + "leave": [] + }} + }} + "# + ); + + let configuration = unindent::unindent( + r#" + { + "update": { + "deadbeef": 2, + "deadca1f": "3", + "deadca2f": false, + "deadca3f": "string0", + "deadca4f": "new_field", + "not_on_whitelist1": "not_on_whitelist", + "on_both_lists": "on_both_lists2" + }, + "delete": ["deadca5f", "not_on_whitelist2", "on_both_lists"] + } + "#, + ); + + let old_config = serde_json::from_str::(&config_json).unwrap(); + + let new_config = super::generate_config_json_migration( + &serde_json::from_str(&schema).unwrap(), + &serde_json::from_str(&configuration).unwrap(), + &old_config, + ) + .unwrap(); + + assert_eq!(new_config.get("deadbeef").unwrap(), 2); + assert_eq!(new_config.get("deadca1f").unwrap(), "3"); + assert_eq!(new_config.get("deadca2f").unwrap(), false); + assert_eq!(new_config.get("deadca3f").unwrap(), "string0"); + assert_eq!(new_config.get("deadca4f").unwrap(), "new_field"); + assert!(new_config.get("deadca5f").is_none()); + assert!(new_config.get("not_on_whitelist1").is_none()); + assert!(new_config.get("not_on_whitelist2").is_none()); + assert!(new_config.get("on_both_lists").is_none()); + } +} diff --git a/tests/integration.rs b/tests/integration.rs index 571932e..8f26f0a 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -140,6 +140,7 @@ fn join() { r#" Fetching service configuration from http://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... Writing {tmp_dir_path}/config.json @@ -318,6 +319,7 @@ fn join_flasher() { r#" Fetching service configuration from http://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... Writing {tmp_dir_path}/config.json @@ -456,6 +458,7 @@ fn join_with_root_certificate() { r#" Fetching service configuration from https://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... @@ -678,6 +681,7 @@ fn reconfigure() { r#" Fetching service configuration from http://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... @@ -826,6 +830,7 @@ fn reconfigure_stored() { r#" Fetching service configuration from http://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... @@ -1004,6 +1009,7 @@ fn update() { r#" Fetching service configuration from http://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... Stopping balena-supervisor.service... Awaiting balena-supervisor.service to exit... {tmp_dir_path}/not-a-service-1.conf updated @@ -1170,6 +1176,7 @@ fn update_no_config_changes() { r#" Fetching service configuration from http://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes "#, )); @@ -1275,6 +1282,7 @@ fn update_with_root_certificate() { r#" Fetching service configuration from https://localhost:{port}/os/v2/config... Service configuration retrieved + Checking for config.json migrations... No configuration changes "#, )); @@ -1829,6 +1837,162 @@ fn generate_api_key_new() { ); } +#[test] +#[timeout(10000)] +fn migrate_config_json() { + let port = 31014; + let tmp_dir = TempDir::new().unwrap(); + let tmp_dir_path = tmp_dir.path().to_str().unwrap().to_string(); + + let config_json = format!( + r#" + {{ + "deviceApiKey": "abcdef", + "deviceType": "intel-nuc", + "hostname": "balena", + "persistentLogging": false, + "applicationName": "aaaaaa", + "applicationId": 1234567, + "userId": 654321, + "appUpdatePollInterval": 900000, + "listenPort": 48484, + "vpnPort": 443, + "apiEndpoint": "http://{}", + "vpnEndpoint": "vpn.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta.balenadev.io", + "apiKey": "12345678abcd1234efgh1234567890ab", + "version": "9.99.9+rev1", + "deadbeef": "12345678", + "mixpanelToken": "12345678abcd1234efgh1234567890ab" + }} + "#, + server_address(port) + ); + + let config_json_path = create_tmp_file(&tmp_dir, "config.json", &config_json, None); + + let schema = format!( + r#" + {{ + "services": [ + ], + "config": {{ + "whitelist": [ + "logsEndpoint", + "mixpanelToken", + "registryEndpoint", + "deltaEndpoint", + "applicationId", + "persistentLogging", + "deadbeef" + ], + "leave": ["apiKey", "apiEndpoint", "vpnEndpoint"] + }} + }} + "# + ); + + let os_config_path = create_tmp_file(&tmp_dir, "os-config.json", &schema, None); + + // Update: + // - apiEndpoint: not on whitelist, should skip + // - logsEndpoint: value not present, should insert + // - registryEndpoint: value unchanged, should skip + // - deltaEndpoint: value changed, should update (String) + // - applicationId: value changed, should update (u64) + // - persistentLogging: value changed, should update (bool) + // - deadbeef: value changed, should update (stringified u64) + // Delete: + // - apiKey: not on whitelist, should skip + // - mixpanelToken: on whitelist, should delete + let configuration = unindent::unindent( + r#" + { + "services": {}, + "config": { + "update": { + "apiEndpoint": "http://api.balenadev.io", + "logsEndpoint": "https://logs.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta2.balenadev.io", + "applicationId": 1234568, + "persistentLogging": true, + "deadbeef": "1234567890" + }, + "delete": ["apiKey", "mixpanelToken"] + } + } + "#, + ); + + let mut serve = serve_config(configuration, false, port); + + // This is unfortunately necessary because the output for each line that + // begins with `Key` may vary in order. + let output_patterns = [ + format!("Fetching service configuration from http://localhost:{}/os/v2/config...\nService configuration retrieved\nChecking for config.json migrations...", port), + "Key `logsEndpoint` not found, will insert".into(), + "Key `deltaEndpoint` found with current value `\"https://delta.balenadev.io\"`, will update to `\"https://delta2.balenadev.io\"`".into(), + "Key `deadbeef` found with current value `\"12345678\"`, will update to `\"1234567890\"`".into(), + "Key `registryEndpoint` found with current value `\"registry2.balenadev.io\"` equal to update value `\"registry2.balenadev.io\"`, skipping".into(), + "Key `persistentLogging` found with current value `false`, will update to `true`".into(), + "Key `apiEndpoint` not in whitelist, skipping".into(), + "Key `applicationId` found with current value `1234567`, will update to `1234568`".into(), + "Key `apiKey` not in whitelist, skipping".into(), + "Key `mixpanelToken` found, will delete".into(), + format!("Stopping balena-supervisor.service...\nAwaiting balena-supervisor.service to exit...\nMigrating config.json...\nWriting {}/config.json\nStarting balena-supervisor.service...", tmp_dir_path), + ]; + + let output = get_base_command() + .args(["update"]) + .timeout(Duration::from_secs(5)) + .envs(os_config_env(&os_config_path, &config_json_path)) + .assert() + .success() + .get_output() + .to_owned(); + + let string_output = String::from_utf8(output.stdout).unwrap(); + + for pattern in output_patterns.iter() { + // write string_output to local file for debugging + assert!(string_output.contains(pattern)); + } + + validate_json_file( + &config_json_path, + &format!( + r#" + {{ + "deviceApiKey": "abcdef", + "deviceType": "intel-nuc", + "hostname": "balena", + "persistentLogging": true, + "applicationName": "aaaaaa", + "applicationId": 1234568, + "userId": 654321, + "appUpdatePollInterval": 900000, + "listenPort": 48484, + "vpnPort": 443, + "apiEndpoint": "http://{}", + "vpnEndpoint": "vpn.balenadev.io", + "registryEndpoint": "registry2.balenadev.io", + "deltaEndpoint": "https://delta2.balenadev.io", + "apiKey": "12345678abcd1234efgh1234567890ab", + "version": "9.99.9+rev1", + "deadbeef": "1234567890", + "logsEndpoint": "https://logs.balenadev.io" + }} + "#, + server_address(port) + ), + false, + ); + + serve.stop(); +} + /******************************************************************************* * os-config launch */