feat(import): allow using multiple worker

backend/Cargo.toml

@@ -11,9 +11,9 @@ members = [
 ]
 
 [patch.crates-io]
-csaf-walker = { git = "https://github.com/ctron/csaf-walker", rev = "c96c96f2f2dff240c394c065354740e7208f4ee1" }
-sbom-walker = { git = "https://github.com/ctron/csaf-walker", rev = "c96c96f2f2dff240c394c065354740e7208f4ee1" }
-walker-common = { git = "https://github.com/ctron/csaf-walker", rev = "c96c96f2f2dff240c394c065354740e7208f4ee1" }
+csaf-walker = { git = "https://github.com/ctron/csaf-walker", rev = "95d1d1e25a0def07e563f6a92722204cee861ea4" }
+sbom-walker = { git = "https://github.com/ctron/csaf-walker", rev = "95d1d1e25a0def07e563f6a92722204cee861ea4" }
+walker-common = { git = "https://github.com/ctron/csaf-walker", rev = "95d1d1e25a0def07e563f6a92722204cee861ea4" }
 
 #csaf-walker = { path = "../../csaf-walker/csaf" }
 #sbom-walker = { path = "../../csaf-walker/sbom" }

backend/importer/src/csaf/mod.rs

@@ -1,28 +1,21 @@
 use crate::progress::init_log_and_progress;
-use ::csaf::document::Category;
-use ::csaf::Csaf;
-use csaf_walker::retrieve::RetrievingVisitor;
-use csaf_walker::source::{DispatchSource, FileSource, HttpSource};
-use csaf_walker::validation::{ValidatedAdvisory, ValidationError, ValidationVisitor};
-use csaf_walker::visitors::filter::{FilterConfig, FilteringVisitor};
-use csaf_walker::walker::Walker;
-use indicatif::MultiProgress;
-use indicatif_log_bridge::LogWrapper;
+use ::csaf::{document::Category, Csaf};
+use csaf_walker::{
+    retrieve::RetrievingVisitor,
+    source::{DispatchSource, FileSource, HttpSource},
+    validation::{ValidatedAdvisory, ValidationError, ValidationVisitor},
+    visitors::filter::{FilterConfig, FilteringVisitor},
+    walker::Walker,
+};
 use sha2::{Digest, Sha256};
 use std::collections::HashSet;
-use std::io::IsTerminal;
 use std::process::ExitCode;
 use std::time::SystemTime;
 use time::{Date, Month, UtcOffset};
-use trustify_api::db::Transactional;
-use trustify_api::system::InnerSystem;
+use trustify_api::{db::Transactional, system::InnerSystem};
 use trustify_common::config::Database;
 use url::Url;
-use walker_common::fetcher::Fetcher;
-use walker_common::progress::indicatif::MultiIndicatif;
-use walker_common::progress::{NoProgress, Progress};
-use walker_common::utils::hex::Hex;
-use walker_common::validate::ValidationOptions;
+use walker_common::{fetcher::Fetcher, utils::hex::Hex, validate::ValidationOptions};
 
 /// Run the importer
 #[derive(clap::Args, Debug)]
@@ -34,16 +27,19 @@ pub struct ImportCsafCommand {
     pub source: String,
 
     /// If the source is a full source URL
-    #[arg(long)]
+    #[arg(long, env)]
     pub full_source_url: bool,
 
     /// Distribution URLs or ROLIE feed URLs to skip
-    #[arg(long)]
+    #[arg(long, env)]
     pub skip_url: Vec<String>,
 
     /// Only consider files having any of those prefixes. An empty list will accept all files.
-    #[arg(long)]
+    #[arg(long, env)]
     pub only_prefix: Vec<String>,
+
+    #[arg(long, env, default_value_t = 1)]
+    pub workers: usize,
 }
 
 impl ImportCsafCommand {
@@ -122,7 +118,7 @@ impl ImportCsafCommand {
             });
         }
 
-        walker.walk(visitor).await?;
+        walker.walk_parallel(self.workers, visitor).await?;
 
         Ok(ExitCode::SUCCESS)
     }

backend/importer/src/progress.rs

@@ -1,8 +1,7 @@
 use indicatif::MultiProgress;
 use indicatif_log_bridge::LogWrapper;
 use std::io::IsTerminal;
-use walker_common::progress::indicatif::MultiIndicatif;
-use walker_common::progress::{NoProgress, Progress};
+use walker_common::progress::Progress;
 
 /// Set up the env_logger and attach a progress interface if we are running on a terminal.
 pub(crate) fn init_log_and_progress() -> Progress {
@@ -19,8 +18,8 @@ pub(crate) fn init_log_and_progress() -> Progress {
             log::set_boxed_logger(Box::new(log)).unwrap();
             log::set_max_level(max_level);
 
-            Progress::new(MultiIndicatif(multi))
+            multi.into()
         }
-        false => Progress::new(NoProgress),
+        false => Progress::default(),
     }
 }