From d84e33a84ed23d275789674067edec947793bb37 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 15 Nov 2024 22:29:39 +0545 Subject: [PATCH] parallel iter --- Cargo.lock | 53 +++++++++++++++++++++++++++++++++++++ squishy-cli/Cargo.toml | 3 ++- squishy-cli/src/appimage.rs | 23 +++++++++------- squishy-cli/src/main.rs | 43 +++++++++++++++++++++--------- squishy/Cargo.toml | 5 ++++ squishy/src/lib.rs | 34 ++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40bf87b..94b1574 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,6 +157,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "darling" version = "0.20.10" @@ -217,6 +242,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "equivalent" version = "1.0.1" @@ -403,6 +434,26 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -446,6 +497,7 @@ name = "squishy" version = "0.2.0" dependencies = [ "backhand", + "rayon", "thiserror 2.0.0", ] @@ -455,6 +507,7 @@ version = "0.2.0" dependencies = [ "clap", "goblin", + "rayon", "squishy", ] diff --git a/squishy-cli/Cargo.toml b/squishy-cli/Cargo.toml index b4d37f9..425833e 100644 --- a/squishy-cli/Cargo.toml +++ b/squishy-cli/Cargo.toml @@ -13,6 +13,7 @@ name = "squishy" path = "src/main.rs" [dependencies] -squishy = { path = "../squishy", version = "0.2.0" } +squishy = { path = "../squishy", version = "0.2.0", features = ["rayon"] } clap = { version = "4.5.20", features = ["cargo", "derive"] } goblin = { version = "0.9.2", default-features = false, features = ["elf32", "elf64", "endian_fd", "std"] } +rayon = "1.10.0" diff --git a/squishy-cli/src/appimage.rs b/squishy-cli/src/appimage.rs index 6e49fb6..9def433 100644 --- a/squishy-cli/src/appimage.rs +++ b/squishy-cli/src/appimage.rs @@ -1,5 +1,10 @@ -use std::{ffi::{OsStr, OsString}, fs, path::Path}; +use std::{ + ffi::{OsStr, OsString}, + fs, + path::Path, +}; +use rayon::iter::ParallelIterator; use squishy::{error::SquishyError, EntryKind, SquashFS, SquashFSEntry}; use crate::common::get_offset; @@ -44,8 +49,8 @@ impl<'a> AppImage<'a> { fn search_diricon(&self) -> Option { self.squashfs - .entries() - .find(|entry| entry.path.to_string_lossy() == "/.DirIcon") + .par_entries() + .find_first(|entry| entry.path.to_string_lossy() == "/.DirIcon") } fn filter_path(&self, path: &str) -> bool { @@ -55,7 +60,7 @@ impl<'a> AppImage<'a> { } fn find_largest_icon_path(&self) -> Option { - let png_entries = self.squashfs.entries().filter(|entry| { + let png_entries = self.squashfs.par_entries().filter(|entry| { let path = entry.path.to_string_lossy().to_lowercase(); path.starts_with("/usr/share/icons/") && self.filter_path(&path) @@ -66,7 +71,7 @@ impl<'a> AppImage<'a> { return Some(entry); } - self.squashfs.entries().find(|entry| { + self.squashfs.par_entries().find_first(|entry| { let path = entry.path.to_string_lossy().to_lowercase(); path.starts_with("/usr/share/icons") && self.filter_path(&path) @@ -75,7 +80,7 @@ impl<'a> AppImage<'a> { } fn find_png_icon(&self) -> Option { - let png_entries = self.squashfs.entries().filter(|entry| { + let png_entries = self.squashfs.par_entries().filter(|entry| { let p = entry.path.to_string_lossy().to_lowercase(); self.filter_path(&p) && p.ends_with(".png") }); @@ -86,14 +91,14 @@ impl<'a> AppImage<'a> { } fn find_svg_icon(&self) -> Option { - self.squashfs.entries().find(|entry| { + self.squashfs.par_entries().find_first(|entry| { let path = entry.path.to_string_lossy().to_lowercase(); self.filter_path(&path) && path.ends_with(".svg") }) } pub fn find_desktop(&self) -> Option { - let desktop = self.squashfs.entries().find(|entry| { + let desktop = self.squashfs.par_entries().find_first(|entry| { let path = entry.path.to_string_lossy().to_lowercase(); self.filter_path(&path) && path.ends_with(".desktop") }); @@ -108,7 +113,7 @@ impl<'a> AppImage<'a> { } pub fn find_appstream(&self) -> Option { - let appstream = self.squashfs.entries().find(|entry| { + let appstream = self.squashfs.par_entries().find_first(|entry| { let path = entry.path.to_string_lossy().to_lowercase(); self.filter_path(&path) && (path.ends_with("appdata.xml") || path.ends_with("metadata.xml")) diff --git a/squishy-cli/src/main.rs b/squishy-cli/src/main.rs index a26a0d6..58d963e 100644 --- a/squishy-cli/src/main.rs +++ b/squishy-cli/src/main.rs @@ -1,9 +1,13 @@ -use std::{fs, os::unix}; +use std::{ + fs::{self, Permissions}, + os::unix::{self, fs::PermissionsExt}, +}; use appimage::AppImage; use clap::Parser; use cli::Args; use common::get_offset; +use rayon::iter::ParallelIterator; use squishy::{error::SquishyError, EntryKind, SquashFS}; mod appimage; @@ -132,40 +136,55 @@ fn main() { }) .unwrap(); - squashfs.entries().for_each(|entry| { + squashfs.par_entries().for_each(|entry| { if let Some(output_dir) = &write_path { let file_path = entry.path.strip_prefix("/").unwrap_or(&entry.path); + let output_path = output_dir.join(file_path); + fs::create_dir_all(output_path.parent().unwrap()).unwrap(); + match entry.kind { EntryKind::File(basic_file) => { - let output_path = output_dir.join(file_path); - squashfs - .write_file_with_permissions(basic_file, &output_path, entry.header) - .unwrap(); + if output_path.exists() { + return; + } + let _ = squashfs.write_file_with_permissions( + basic_file, + &output_path, + entry.header, + ); log!( args.quiet, "Wrote {} to {}", - file.display(), + entry.path.display(), output_path.display() ); } EntryKind::Directory => { - let output_path = output_dir.join(file_path); + if output_path.exists() { + return; + } fs::create_dir_all(&output_path).unwrap(); + fs::set_permissions( + &output_path, + Permissions::from_mode(u32::from(entry.header.permissions)), + ).unwrap(); log!( args.quiet, "Wrote {} to {}", - file.display(), + entry.path.display(), output_path.display() ); } EntryKind::Symlink(e) => { + if output_path.exists() { + return; + } let original_path = e.strip_prefix("/").unwrap_or(&e); - let output_path = output_dir.join(file_path); - unix::fs::symlink(original_path, &output_path).unwrap(); + let _ = unix::fs::symlink(original_path, &output_path); log!( args.quiet, "Wrote {} to {}", - file.display(), + entry.path.display(), output_path.display() ); } diff --git a/squishy/Cargo.toml b/squishy/Cargo.toml index 264c9c8..dcd8338 100644 --- a/squishy/Cargo.toml +++ b/squishy/Cargo.toml @@ -12,6 +12,11 @@ keywords.workspace = true name = "squishy" path = "src/lib.rs" +[features] +default = [] +rayon = ["dep:rayon"] + [dependencies] backhand = "0.18.0" +rayon = { version = "1.10.0", optional = true } thiserror = "2.0.0" diff --git a/squishy/src/lib.rs b/squishy/src/lib.rs index a5cdefb..7277b41 100644 --- a/squishy/src/lib.rs +++ b/squishy/src/lib.rs @@ -9,6 +9,9 @@ use std::{ use backhand::{kind::Kind, BasicFile, FilesystemReader, InnerNode, NodeHeader}; use error::SquishyError; +#[cfg(feature = "rayon")] +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + pub mod error; pub type Result = std::result::Result; @@ -135,6 +138,37 @@ impl<'a> SquashFS<'a> { }) } + #[cfg(feature = "rayon")] + /// Returns a parallel iterator over all the entries in the SquashFS filesystem. + pub fn par_entries(&self) -> impl ParallelIterator + '_ { + self.reader + .files() + .map(|node| { + let size = match &node.inner { + InnerNode::File(file) => file.basic.file_size, + _ => 0, + }; + + let kind = match &node.inner { + InnerNode::File(file) => EntryKind::File(&file.basic), + InnerNode::Dir(_) => EntryKind::Directory, + InnerNode::Symlink(symlink) => EntryKind::Symlink( + PathBuf::from(format!("/{}", symlink.link.display())).clone(), + ), + _ => EntryKind::Unknown, + }; + + SquashFSEntry { + header: node.header, + path: node.fullpath.clone(), + size, + kind, + } + }) + .collect::>() + .into_par_iter() + } + /// Returns an iterator over all the entries in the SquashFS filesystem /// that match the provided predicate function. ///