Merge pull request #72 from balena-os/cywang117/os-config-v2

Allow migrating of whitelisted config.json fields

src/join.rs

@@ -6,7 +6,8 @@ use crate::config_json::{
     get_api_endpoint, get_root_certificate, merge_config_json, read_config_json, write_config_json,
     ConfigMap,
 };
-use crate::remote::{config_url, fetch_configuration, Configuration};
+use crate::migrate::migrate_config_json;
+use crate::remote::{config_url, fetch_configuration, RemoteConfiguration};
 use crate::schema::{read_os_config_schema, OsConfigSchema};
 use crate::systemd;
 use anyhow::Result;
@@ -24,10 +25,10 @@ pub fn join(args: &Args) -> Result<()> {
         unreachable!()
     };
 
-    reconfigure(args, &config_json, true)
+    reconfigure(args, &mut config_json, true)
 }
 
-pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Result<()> {
+pub fn reconfigure(args: &Args, config_json: &mut ConfigMap, joining: bool) -> Result<()> {
     let schema = read_os_config_schema(&args.os_config_path)?;
 
     let api_endpoint = if let Some(api_endpoint) = get_api_endpoint(config_json)? {
@@ -39,15 +40,18 @@ pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Resul
 
     let root_certificate = get_root_certificate(config_json)?;
 
-    let configuration = fetch_configuration(
+    let remote_config = fetch_configuration(
         &config_url(&api_endpoint, &args.config_route),
         root_certificate,
         !joining,
     )?;
 
-    let has_config_changes = has_config_changes(&schema, &configuration)?;
+    let has_service_config_changes = has_service_config_changes(&schema, &remote_config)?;
 
-    if !has_config_changes {
+    let has_config_json_migrations =
+        migrate_config_json(&schema, &remote_config.config, config_json);
+
+    if !has_service_config_changes && !has_config_json_migrations {
         info!("No configuration changes");
 
         if !joining {
@@ -61,13 +65,15 @@ pub fn reconfigure(args: &Args, config_json: &ConfigMap, joining: bool) -> Resul
         systemd::await_service_exit(SUPERVISOR_SERVICE)?;
     }
 
+    let should_write_config_json = joining || has_config_json_migrations;
+
     let result = reconfigure_core(
         args,
         config_json,
         &schema,
-        &configuration,
-        has_config_changes,
-        joining,
+        &remote_config,
+        has_service_config_changes,
+        should_write_config_json,
     );
 
     if args.supervisor_exists {
@@ -81,25 +87,28 @@ fn reconfigure_core(
     args: &Args,
     config_json: &ConfigMap,
     schema: &OsConfigSchema,
-    configuration: &Configuration,
-    has_config_changes: bool,
-    joining: bool,
+    remote_config: &RemoteConfiguration,
+    has_service_config_changes: bool,
+    should_write_config_json: bool,
 ) -> Result<()> {
-    if joining {
+    if should_write_config_json {
         write_config_json(&args.config_json_path, config_json)?;
     }
 
-    if has_config_changes {
-        configure_services(schema, configuration)?;
+    if has_service_config_changes {
+        configure_services(schema, remote_config)?;
     }
 
     Ok(())
 }
 
-fn has_config_changes(schema: &OsConfigSchema, configuration: &Configuration) -> Result<bool> {
+fn has_service_config_changes(
+    schema: &OsConfigSchema,
+    remote_config: &RemoteConfiguration,
+) -> Result<bool> {
     for service in &schema.services {
         for (name, config_file) in &service.files {
-            let future = configuration.get_config_contents(&service.id, name)?;
+            let future = remote_config.get_config_contents(&service.id, name)?;
             let current = get_config_contents(&config_file.path);
 
             if future != current {
@@ -111,7 +120,7 @@ fn has_config_changes(schema: &OsConfigSchema, configuration: &Configuration) ->
     Ok(false)
 }
 
-fn configure_services(schema: &OsConfigSchema, configuration: &Configuration) -> Result<()> {
+fn configure_services(schema: &OsConfigSchema, remote_config: &RemoteConfiguration) -> Result<()> {
     for service in &schema.services {
         for systemd_service in &service.systemd_services {
             systemd::stop_service(systemd_service)?;
@@ -126,7 +135,7 @@ fn configure_services(schema: &OsConfigSchema, configuration: &Configuration) ->
         names.sort();
         for name in names {
             let config_file = &service.files[name as &str];
-            let contents = configuration.get_config_contents(&service.id, name)?;
+            let contents = remote_config.get_config_contents(&service.id, name)?;
             let mode = fs::parse_mode(&config_file.perm)?;
             fs::write_file(Path::new(&config_file.path), contents, mode)?;
             info!("{} updated", &config_file.path);

src/main.rs

@@ -33,6 +33,7 @@ mod generate;
 mod join;
 mod leave;
 mod logger;
+mod migrate;
 mod random;
 mod remote;
 mod schema;

src/migrate.rs

@@ -0,0 +1,130 @@
+// Config.json migration module
+//
+// Provides methods for migrating config.json fields based on remote directives
+// from /os/vX/config. Limits migrated fields based on os-config.json schema
+// whitelist.
+
+use crate::config_json::ConfigMap;
+use crate::remote::{ConfigMigrationInstructions, OverridesMap};
+use crate::schema::OsConfigSchema;
+
+pub fn migrate_config_json(
+    schema: &OsConfigSchema,
+    migration: &ConfigMigrationInstructions,
+    config_json: &mut ConfigMap,
+) -> bool {
+    info!("Checking for config.json migrations...");
+
+    let overridden = handle_override_directives(schema, &migration.overrides, config_json);
+
+    if overridden {
+        info!("Done config.json migrations");
+    }
+
+    overridden
+}
+
+fn handle_override_directives(
+    schema: &OsConfigSchema,
+    overrides: &OverridesMap,
+    config_json: &mut ConfigMap,
+) -> bool {
+    let mut overridden = false;
+
+    // Sort overrides by key in order for tests to have predictable order
+    let mut items = overrides.iter().collect::<Vec<_>>();
+    items.sort_by_key(|pair| pair.0);
+
+    for (key, new_value) in items {
+        if !schema.config.whitelist.contains(key) {
+            info!("Key `{}` not in whitelist, skipping", key);
+            continue;
+        }
+
+        if let Some(existing_value) = config_json.get_mut(key) {
+            if new_value != existing_value {
+                info!(
+                    "Key `{}` found with existing value `{}`, will override to `{}`",
+                    key, existing_value, new_value
+                );
+                *existing_value = new_value.clone();
+                overridden = true;
+            } else {
+                debug!(
+                    "Key `{}` found with existing value `{}` equal to override value `{}`, skipping",
+                    key, existing_value, new_value
+                );
+            }
+        } else {
+            info!("Key `{}` not found, will insert `{}`", key, new_value);
+            config_json.insert(key.to_string(), new_value.clone());
+            overridden = true;
+        }
+    }
+
+    overridden
+}
+
+mod tests {
+    #[test]
+    fn test_generate_config_json_migration() {
+        let config_json = r#"
+            {
+                "deadbeef": 1,
+                "deadca1f": "2",
+                "deadca2f": true,
+                "deadca3f": "string1"
+            }
+        "#
+        .to_string();
+
+        let schema = r#"
+            {
+                "services": [
+                ],
+                "keys": [],
+                "config": {
+                    "whitelist": [
+                        "deadbeef",
+                        "deadca1f",
+                        "deadca2f",
+                        "deadca3f",
+                        "deadca4f"
+                    ]
+                }
+            }
+            "#
+        .to_string();
+
+        let configuration = unindent::unindent(
+            r#"
+            {
+                "overrides": {
+                    "deadbeef": 2,
+                    "deadca1f": "3",
+                    "deadca2f": false,
+                    "deadca3f": "string0",
+                    "deadca4f": "new_field",
+                    "not_on_whitelist1": "not_on_whitelist"
+                }
+            }
+            "#,
+        );
+
+        let mut config = serde_json::from_str::<super::ConfigMap>(&config_json).unwrap();
+
+        let has_config_json_migrations = super::migrate_config_json(
+            &serde_json::from_str(&schema).unwrap(),
+            &serde_json::from_str(&configuration).unwrap(),
+            &mut config,
+        );
+
+        assert!(has_config_json_migrations);
+        assert_eq!(config.get("deadbeef").unwrap(), 2);
+        assert_eq!(config.get("deadca1f").unwrap(), "3");
+        assert_eq!(config.get("deadca2f").unwrap(), false);
+        assert_eq!(config.get("deadca3f").unwrap(), "string0");
+        assert_eq!(config.get("deadca4f").unwrap(), "new_field");
+        assert!(config.get("not_on_whitelist1").is_none());
+    }
+}

src/remote.rs

@@ -2,16 +2,22 @@ use std::collections::HashMap;
 use std::thread;
 use std::time::Duration;
 
-use crate::schema::validate_schema_version;
 use anyhow::{anyhow, Context, Result};
 
+pub type OverridesMap = HashMap<String, serde_json::Value>;
+
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
-pub struct Configuration {
+pub struct RemoteConfiguration {
     pub services: HashMap<String, HashMap<String, String>>,
-    pub schema_version: String,
+    pub config: ConfigMigrationInstructions,
 }
 
-impl Configuration {
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+pub struct ConfigMigrationInstructions {
+    pub overrides: OverridesMap,
+}
+
+impl RemoteConfiguration {
     pub fn get_config_contents<'a>(
         &'a self,
         service_id: &str,
@@ -42,7 +48,7 @@ pub fn fetch_configuration(
     config_url: &str,
     root_certificate: Option<reqwest::Certificate>,
     retry: bool,
-) -> Result<Configuration> {
+) -> Result<RemoteConfiguration> {
     fetch_configuration_impl(config_url, root_certificate, retry)
         .context("Fetching configuration failed")
 }
@@ -51,7 +57,7 @@ fn fetch_configuration_impl(
     config_url: &str,
     root_certificate: Option<reqwest::Certificate>,
     retry: bool,
-) -> Result<Configuration> {
+) -> Result<RemoteConfiguration> {
     let client = build_reqwest_client(root_certificate)?;
 
     let request_fn = if retry {
@@ -66,8 +72,6 @@ fn fetch_configuration_impl(
 
     info!("Service configuration retrieved");
 
-    validate_schema_version(&json_data)?;
-
     Ok(serde_json::from_str(&json_data)?)
 }
 
@@ -141,7 +145,6 @@ fn build_reqwest_client(
 mod tests {
 
     use super::*;
-    use crate::schema::SCHEMA_VERSION;
 
     const JSON_DATA: &str = r#"{
         "services": {
@@ -155,14 +158,18 @@ mod tests {
                 "authorized_keys": "authorized keys here"
             }
         },
-        "schema_version": "1.0.0"
+        "config": {
+            "overrides": {
+                "logsEndpoint": "https://logs.balenadev.io"
+            }
+        }
     }"#;
 
     #[test]
-    fn parse_configuration_v1() {
-        let parsed: Configuration = serde_json::from_str(JSON_DATA).unwrap();
+    fn parse_configuration() {
+        let parsed: RemoteConfiguration = serde_json::from_str(JSON_DATA).unwrap();
 
-        let expected = Configuration {
+        let expected = RemoteConfiguration {
             services: hashmap! {
                 "openvpn".into() => hashmap!{
                     "config".into() => "main configuration here".into(),
@@ -174,7 +181,11 @@ mod tests {
                     "authorized_keys".into() => "authorized keys here".into()
                 }
             },
-            schema_version: SCHEMA_VERSION.into(),
+            config: ConfigMigrationInstructions {
+                overrides: hashmap! {
+                    "logsEndpoint".into() => "https://logs.balenadev.io".into()
+                },
+            },
         };
 
         assert_eq!(parsed, expected);