Merge pull request #604 from rustic-rs/rework-config

rework config implementation

changelog/new.txt

@@ -1,13 +1,16 @@
 Changes in version x.x.x:
 
 Breaking changes:
+- The option `--config-profile` was renamed into `--use-profile`
 
 Bugs fixed:
 - restore: Warm-up options given by the command line didn't work. This has been fixed.
 - backup showed 1 dir as changed when backing up without parent. This has been fixed.
 - diff: The options --no-atime and --ignore-devid had no effect and are now removed.
+- Rustic's check of additional fields in the config file didn't work in edge cases. This has been fixed.
 
 New features:
+- config file: New field use-profile allows to merge options from other config profiles
 - backup: Backing up (small) files is now much more parallelized.
 - forget: Using "-1" as value for --keep-* options will keep all snapshots of that interval
 - prune: Added option --repack-all

examples/full.toml

@@ -7,6 +7,7 @@
 
 # Global options: These options are used for all commands. 
 [global]
+use-profile = "" 
 log-level = "info" # any of "off", "error", "warn", "info", "debug", "trace"; default: "info"
 log-file = "/path/to/rustic.log" # Default: not set
 no-progress = false
@@ -32,7 +33,7 @@ warm-up-wait = "10min" # Default: not set
 [repository.options]
 post-create-command = "par2create -qq -n1 -r5 %file" # Only local backend; Default: not set
 post-delete-command = "sh -c \"rm -f %file*.par2\"" # Only local backend; Default: not set 
-retry = true # Only rest/rclone backend
+retry = "true" # Only rest/rclone backend
 timeout = "2min" # Ony rest/rclone backend
 
 # Snapshot-filter options: These options apply to all commands that use snapshot filters
@@ -114,7 +115,7 @@ filter-host = ["host2", "host2"] # Default: no host filter
 filter-label = ["label1", "label2"] # Default: no label filter
 filter-tags = ["tag1,tag2", "tag3"] # Default: no tags filger
 filter-paths = ["path1", "path2,path3"] # Default: no paths filter
-filter-fn = '|sn| {sn.host == "host1" || sn.description.contains("test")}'' # Default: no filter function
+filter-fn = '|sn| {sn.host == "host1" || sn.description.contains("test")}' # Default: no filter function
 # The retention options follow. All of these are not set by default.
 keep-tags = ["tag1", "tag2,tag3"]
 keep-ids = ["123abc", "11122233"] # Keep all snapshots whose ID starts with any of these strings

src/commands/backup.rs

@@ -8,9 +8,8 @@ use log::*;
 use merge::Merge;
 use path_dedot::ParseDot;
 use serde::Deserialize;
-use toml::Value;
 
-use super::{bytes, progress_bytes, progress_counter, GlobalOpts, RusticConfig};
+use super::{bytes, progress_bytes, progress_counter, Config};
 use crate::archiver::Archiver;
 use crate::backend::{
     DryRunBackend, LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions, StdinSource,
@@ -22,8 +21,13 @@ use crate::repofile::{
 use crate::repository::OpenRepository;
 
 #[derive(Clone, Default, Parser, Deserialize, Merge)]
-#[serde(default, rename_all = "kebab-case")]
-pub(super) struct Opts {
+#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
+// Note: using cli_sources, sources and source within this strict is a hack to support serde(deny_unknown_fields)
+// for deserializing the backup options from TOML
+// Unfortunately we cannot work with nested flattened structures, see
+// https://github.com/serde-rs/serde/issues/1547
+// A drawback is that a wrongly set "source(s) = ..." won't get correct error handling and need to be manually checked, see below.
+pub struct Opts {
     /// Backup source (can be specified multiple times), use - for stdin. If no source is given, uses all
     /// sources defined in the config file
     #[clap(value_name = "SOURCE")]
@@ -103,15 +107,9 @@ pub(super) struct Opts {
     #[merge(strategy = merge::bool::overwrite_false)]
     json: bool,
 
-    // This is a hack to support serde(deny_unknown_fields) for deserializing the backup options from TOML
-    // while still being able to use [[backup.sources]] in the config file.
-    // A drawback is that a unkowen "sources = ..." won't be bailed...
-    // Note that unfortunately we cannot work with nested flattened structures, see
-    // https://github.com/serde-rs/serde/issues/1547
     #[clap(skip)]
     #[merge(skip)]
-    #[serde(rename = "sources")]
-    config_sources: Option<Value>,
+    sources: Vec<Opts>,
 
     /// Backup source, used within config file
     #[clap(skip)]
@@ -121,14 +119,24 @@ pub(super) struct Opts {
 
 pub(super) fn execute(
     repo: OpenRepository,
-    gopts: GlobalOpts,
+    mut config: Config,
     opts: Opts,
-    config_file: RusticConfig,
     command: String,
 ) -> Result<()> {
     let time = Local::now();
 
-    let config_opts: Vec<Opts> = config_file.get("backup.sources")?;
+    // manually check for a "source" field, check is not done by serde, see above.
+    if !config.backup.source.is_empty() {
+        bail!("key \"source\" is not valid in the [backup] section!");
+    }
+
+    let config_opts = config.backup.sources;
+    config.backup.sources = Vec::new();
+
+    // manually check for a "sources" field, check is not done by serde, see above.
+    if config_opts.iter().any(|opt| !opt.sources.is_empty()) {
+        bail!("key \"sources\" is not valid in a [[backup.sources]] section!");
+    }
 
     let config_sources: Vec<_> = config_opts
         .iter()
@@ -186,10 +194,11 @@ pub(super) fn execute(
                 }
             }
         }
+
         // merge "backup" section from config file, if given
-        config_file.merge_into("backup", &mut opts)?;
+        opts.merge(config.backup.clone());
 
-        let be = DryRunBackend::new(repo.dbe.clone(), gopts.dry_run);
+        let be = DryRunBackend::new(repo.dbe.clone(), config.global.dry_run);
         info!("starting to backup {source}...");
         let as_path = match opts.as_path {
             None => None,

src/commands/cat.rs

@@ -4,13 +4,12 @@ use anyhow::{anyhow, Result};
 use clap::{Parser, Subcommand};
 use indicatif::ProgressBar;
 
-use super::progress_counter;
-use super::rustic_config::RusticConfig;
+use super::{progress_counter, Config};
 use crate::backend::{DecryptReadBackend, FileType};
 use crate::blob::{BlobType, Tree};
 use crate::id::Id;
 use crate::index::{IndexBackend, IndexedBackend};
-use crate::repofile::{SnapshotFile, SnapshotFilter};
+use crate::repofile::SnapshotFile;
 use crate::repository::OpenRepository;
 
 #[derive(Parser)]
@@ -46,15 +45,9 @@ struct TreeOpts {
     /// Snapshot/path of the tree to display
     #[clap(value_name = "SNAPSHOT[:PATH]")]
     snap: String,
-
-    #[clap(
-        flatten,
-        next_help_heading = "Snapshot filter options (when using latest)"
-    )]
-    filter: SnapshotFilter,
 }
 
-pub(super) fn execute(repo: OpenRepository, opts: Opts, config_file: RusticConfig) -> Result<()> {
+pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
     let be = &repo.dbe;
     match opts.command {
         Command::Config => cat_file(be, FileType::Config, IdOpt::default()),
@@ -64,7 +57,7 @@ pub(super) fn execute(repo: OpenRepository, opts: Opts, config_file: RusticConfi
         Command::TreeBlob(opt) => cat_blob(be, BlobType::Tree, opt),
         Command::DataBlob(opt) => cat_blob(be, BlobType::Data, opt),
         // special treatment for cating a tree within a snapshot
-        Command::Tree(opts) => cat_tree(be, opts, config_file),
+        Command::Tree(opts) => cat_tree(be, config, opts),
     }
 }
 
@@ -84,15 +77,14 @@ fn cat_blob(be: &impl DecryptReadBackend, tpe: BlobType, opt: IdOpt) -> Result<(
     Ok(())
 }
 
-fn cat_tree(
-    be: &impl DecryptReadBackend,
-    mut opts: TreeOpts,
-    config_file: RusticConfig,
-) -> Result<()> {
-    config_file.merge_into("snapshot-filter", &mut opts.filter)?;
-
+fn cat_tree(be: &impl DecryptReadBackend, config: Config, opts: TreeOpts) -> Result<()> {
     let (id, path) = opts.snap.split_once(':').unwrap_or((&opts.snap, ""));
-    let snap = SnapshotFile::from_str(be, id, |sn| sn.matches(&opts.filter), progress_counter(""))?;
+    let snap = SnapshotFile::from_str(
+        be,
+        id,
+        |sn| sn.matches(&config.snapshot_filter),
+        progress_counter(""),
+    )?;
     let index = IndexBackend::new(be, progress_counter(""))?;
     let node = Tree::node_from_path(&index, snap.tree, Path::new(path))?;
     let id = node.subtree.ok_or_else(|| anyhow!("{path} is no dir"))?;

src/commands/completions.rs

@@ -26,7 +26,7 @@ pub(super) fn execute(opts: Opts) {
 }
 
 fn generate_completion<G: Generator>(shell: G, buf: &mut dyn Write) {
-    let mut command = super::Opts::command();
+    let mut command = super::Args::command();
     generate(shell, &mut command, env!("CARGO_BIN_NAME"), buf);
 }
 

src/commands/configfile.rs

@@ -0,0 +1,64 @@
+use std::path::Path;
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use directories::ProjectDirs;
+use merge::Merge;
+use serde::Deserialize;
+
+use crate::{repofile::SnapshotFilter, repository::RepositoryOptions};
+
+use super::{backup, copy, forget, GlobalOpts};
+
+#[derive(Default, Parser, Deserialize, Merge)]
+#[serde(default, rename_all = "kebab-case", deny_unknown_fields)]
+pub struct Config {
+    #[clap(flatten, next_help_heading = "Global options")]
+    pub global: GlobalOpts,
+
+    #[clap(flatten, next_help_heading = "Repository options")]
+    pub repository: RepositoryOptions,
+
+    #[clap(flatten, next_help_heading = "Snapshot filter options")]
+    pub snapshot_filter: SnapshotFilter,
+
+    #[clap(skip)]
+    pub backup: backup::Opts,
+
+    #[clap(skip)]
+    pub copy: copy::Targets,
+
+    #[clap(skip)]
+    pub forget: forget::ConfigOpts,
+}
+
+impl Config {
+    pub fn merge_profile(&mut self, profile: &str) -> Result<()> {
+        let mut path = match ProjectDirs::from("", "", "rustic") {
+            Some(path) => path.config_dir().to_path_buf(),
+            None => Path::new(".").to_path_buf(),
+        };
+        if !path.exists() {
+            path = Path::new(".").to_path_buf();
+        };
+        let path = path.join(profile.to_string() + ".toml");
+
+        if path.exists() {
+            // TODO: This should be log::info! - however, the logging config
+            // can be stored in the config file and is needed to initialize the logger
+            eprintln!("using config {}", path.display());
+            let data = std::fs::read_to_string(path).context("error reading config file")?;
+            let mut config: Self =
+                toml::from_str(&data).context("error reading TOML from config file")?;
+            // if "use_profile" is defined in config file, merge this referenced profile first
+            if !config.global.use_profile.is_empty() {
+                let profile = config.global.use_profile.clone();
+                config.merge_profile(&profile)?;
+            }
+            self.merge(config);
+        } else {
+            eprintln!("using no config file ({} doesn't exist)", path.display());
+        };
+        Ok(())
+    }
+}

src/commands/copy.rs

@@ -3,9 +3,11 @@ use std::collections::BTreeSet;
 use anyhow::{bail, Result};
 use clap::Parser;
 use log::*;
+use merge::Merge;
 use rayon::prelude::*;
+use serde::Deserialize;
 
-use super::{progress_counter, table_with_titles, GlobalOpts, RusticConfig};
+use super::{progress_counter, table_with_titles, Config};
 use crate::backend::DecryptWriteBackend;
 use crate::blob::{BlobType, NodeType, Packer, TreeStreamerOnce};
 use crate::index::{IndexBackend, IndexedBackend, Indexer, ReadIndex};
@@ -17,30 +19,22 @@ pub(super) struct Opts {
     /// Snapshots to copy. If none is given, use filter options to filter from all snapshots.
     #[clap(value_name = "ID")]
     ids: Vec<String>,
-
-    #[clap(
-        flatten,
-        next_help_heading = "Snapshot filter options (if no snapshot is given)"
-    )]
-    filter: SnapshotFilter,
 }
 
-pub(super) fn execute(
-    repo: OpenRepository,
-    gopts: GlobalOpts,
-    mut opts: Opts,
-    config_file: RusticConfig,
-) -> Result<()> {
-    config_file.merge_into("snapshot-filter", &mut opts.filter)?;
+#[derive(Default, Deserialize, Merge)]
+pub struct Targets {
+    #[merge(strategy = merge::vec::overwrite_empty)]
+    targets: Vec<RepositoryOptions>,
+}
 
-    let target_opts: Vec<RepositoryOptions> = config_file.get("copy.targets")?;
-    if target_opts.is_empty() {
+pub(super) fn execute(repo: OpenRepository, config: Config, opts: Opts) -> Result<()> {
+    if config.copy.targets.is_empty() {
         bail!("no [[copy.targets]] section in config file found!");
     }
 
     let be = &repo.dbe;
     let mut snapshots = match opts.ids.is_empty() {
-        true => SnapshotFile::all_from_backend(be, &opts.filter)?,
+        true => SnapshotFile::all_from_backend(be, &config.snapshot_filter)?,
         false => SnapshotFile::from_ids(be, &opts.ids)?,
     };
     // sort for nicer output
@@ -51,13 +45,13 @@ pub(super) fn execute(
 
     let poly = repo.config.poly()?;
 
-    for target_opt in target_opts {
-        let repo_dest = Repository::new(target_opt)?.open()?;
+    for target_opt in &config.copy.targets {
+        let repo_dest = Repository::new(target_opt.clone())?.open()?;
         info!("copying to target {}...", repo_dest.name);
         if poly != repo_dest.config.poly()? {
             bail!("cannot copy to repository with different chunker parameter (re-chunking not implemented)!");
         }
-        copy(&snapshots, index.clone(), repo_dest, &gopts, &opts)?;
+        copy(&snapshots, index.clone(), repo_dest, &config)?;
     }
     Ok(())
 }
@@ -66,13 +60,12 @@ fn copy(
     snapshots: &[SnapshotFile],
     index: impl IndexedBackend,
     repo_dest: OpenRepository,
-    gopts: &GlobalOpts,
-    opts: &Opts,
+    config: &Config,
 ) -> Result<()> {
     let be_dest = &repo_dest.dbe;
 
-    let snapshots = relevant_snapshots(snapshots, &repo_dest, &opts.filter)?;
-    match (snapshots.len(), gopts.dry_run) {
+    let snapshots = relevant_snapshots(snapshots, &repo_dest, &config.snapshot_filter)?;
+    match (snapshots.len(), config.global.dry_run) {
         (count, true) => {
             info!("would have copied {count} snapshots");
             return Ok(());