From 420610297fcc8c82c6afd886cc65492ea587964c Mon Sep 17 00:00:00 2001 From: sokorototo Date: Sat, 30 Mar 2024 00:37:48 +0300 Subject: [PATCH 01/17] added some missing documentation, minimal CLI changes --- vach-cli/README.md | 30 +++++--- vach-cli/src/commands/list.rs | 125 ++++++++++++++-------------------- vach-cli/src/commands/pack.rs | 39 +++++------ 3 files changed, 87 insertions(+), 107 deletions(-) diff --git a/vach-cli/README.md b/vach-cli/README.md index 55e48fdd..eb36241e 100644 --- a/vach-cli/README.md +++ b/vach-cli/README.md @@ -63,9 +63,8 @@ vach list -i textures.vach > `pack` is used to pack files and directories into archives. It takes inputs for customizing how the archive should be packaged. ```sh -# The simplest pack command. # Any pack command must have an output, set using the "-o" or "--output" keys -# This builds only an empty archive +# This builds an empty archive vach pack -o hello.vach # You can add files as inputs using the "-i" or "--input" keys @@ -127,10 +126,9 @@ vach pack -t -o hello.vach -i hello.txt goodbye.txt ### 2: unpack ->`unpack` is used to unpack archives back into their constituent files. +>`unpack` it's just like `pack` but backwards ```sh -# The simplest unpack command # Provide an input: "-i" or "--input" vach unpack -i target.vach @@ -151,12 +149,26 @@ vach unpack -k keypair.kp -i source.vach vach unpack -s keypair.sk -i source.vach ``` -### 3: list +### 3: pipe + +>`pipe`: Read the data from a _specific_ entry and pipe it to stdout + +```sh +# Print to stdout +vach pipe -i target.vach -r npc-dialogue.txt + +# Pipe directly into a file +vach pipe -i target.vach -r npc-dialogue.txt >> npc-dialogue.txt + +# Pipe into another process' stdin +vach pipe -i presets.vach -r low.json | jq '."TextureResolution"' +``` + +### 4: list > Lists all the entries in the archive as a table ```sh -# The simplest list command # Provide some input: "-i" or "--input" vach list -i textures.vach @@ -170,7 +182,7 @@ vach list -i textures.vach -m TXTRS vach list -i textures.vach -m TXTRS --sort size-descending ``` -### 4: verify +### 5: verify > Verifies the validity of a file as an archive @@ -182,7 +194,7 @@ vach verify -i textures.vach vach verify -i textures.vach -m TXTRS ``` -### 5: keypair +### 6: keypair > Key-pair generation command @@ -197,7 +209,7 @@ vach keypair -s -o keypair.kp # -> keypair.sk ``` -### 6: split +### 7: split > Splits an existing keypair into it's public and secret components diff --git a/vach-cli/src/commands/list.rs b/vach-cli/src/commands/list.rs index 6be3810f..a62a1b38 100644 --- a/vach-cli/src/commands/list.rs +++ b/vach-cli/src/commands/list.rs @@ -6,22 +6,14 @@ use tabled::{ }; use vach::{ prelude::{ArchiveConfig, Archive, Flags}, - archive::{CompressionAlgorithm, RegistryEntry}, + archive::RegistryEntry, }; use indicatif::HumanBytes; use super::CommandTrait; use crate::keys::key_names; -pub const VERSION: &str = "0.2.0"; - -enum Sort { - SizeAscending, - SizeDescending, - Alphabetical, - AlphabeticalReversed, - None, -} +pub const VERSION: &str = "0.2.1"; /// This command lists the entries in an archive in tabulated form pub struct Evaluator; @@ -35,15 +27,6 @@ impl CommandTrait for Evaluator { }, }; - let sort = match args.value_of(key_names::SORT) { - Some("alphabetical") => Sort::Alphabetical, - Some("alphabetical-reversed") => Sort::AlphabeticalReversed, - Some("size-ascending") => Sort::SizeAscending, - Some("size-descending") => Sort::SizeDescending, - Some(sort) => anyhow::bail!("Unknown sort given: {}. Valid sort types are: 'alphabetical' 'alphabetical-descending' 'size-ascending' 'size-descending'", sort), - None => Sort::None, - }; - let magic: [u8; vach::MAGIC_LENGTH] = match args.value_of(key_names::MAGIC) { Some(magic) => magic.as_bytes().try_into()?, None => *vach::DEFAULT_MAGIC, @@ -52,61 +35,53 @@ impl CommandTrait for Evaluator { let file = File::open(archive_path)?; let archive = Archive::with_config(file, &ArchiveConfig::new(magic, None))?; - if archive.entries().is_empty() { - println!("{}", archive); - } else { - let mut entries: Vec<(String, RegistryEntry)> = archive - .entries() - .iter() - .map(|(id, entry)| (id.clone(), entry.clone())) - .collect(); - - // Log some additional info about this archive - println!("{}", archive); - - // Sort the entries accordingly - match sort { - Sort::SizeAscending => entries.sort_by(|a, b| a.1.offset.cmp(&b.1.offset)), - Sort::SizeDescending => entries.sort_by(|a, b| b.1.offset.cmp(&a.1.offset)), - Sort::Alphabetical => entries.sort_by(|a, b| a.0.cmp(&b.0)), - Sort::AlphabeticalReversed => entries.sort_by(|a, b| b.0.cmp(&a.0)), - Sort::None => (), - }; - - let table_entries: Vec = entries - .iter() - .map(|(id, entry)| { - let c_algo = if entry.flags.contains(Flags::LZ4_COMPRESSED) { - Some(CompressionAlgorithm::LZ4) - } else if entry.flags.contains(Flags::BROTLI_COMPRESSED) { - Some(CompressionAlgorithm::Brotli(8)) - } else if entry.flags.contains(Flags::SNAPPY_COMPRESSED) { - Some(CompressionAlgorithm::Snappy) - } else { - None - }; - - let c_algo = match c_algo { - Some(algo) => algo.to_string(), - None => "None".to_string(), - }; - - FileTableEntry { - id, - size: HumanBytes(entry.offset).to_string(), - flags: entry.flags, - compression: c_algo, - } - }) - .collect(); - - let mut table = Table::new(table_entries); - table - .with(Style::rounded()) - .with(Modify::list(Columns::new(..1), Alignment::left())); - - println!("{}", table); - } + // log basic metadata + println!("{}", archive); + + let mut entries: Vec<(String, RegistryEntry)> = archive + .entries() + .iter() + .map(|(id, entry)| (id.clone(), entry.clone())) + .collect(); + + // Sort the entries accordingly + match args.value_of(key_names::SORT) { + Some("alphabetical") => entries.sort_by(|a, b| a.0.cmp(&b.0)), + Some("alphabetical-reversed") => entries.sort_by(|a, b| b.0.cmp(&a.0)), + Some("size-ascending") => entries.sort_by(|a, b| a.1.offset.cmp(&b.1.offset)), + Some("size-descending") => entries.sort_by(|a, b| b.1.offset.cmp(&a.1.offset)), + Some(sort) => anyhow::bail!("Unknown sort option provided: {}. Valid sort types are: 'alphabetical' 'alphabetical-descending' 'size-ascending' 'size-descending'", sort), + _ => (), + }; + + let table_entries: Vec = entries + .iter() + .map(|(id, entry)| { + let c_algo = if entry.flags.contains(Flags::LZ4_COMPRESSED) { + "LZ4" + } else if entry.flags.contains(Flags::BROTLI_COMPRESSED) { + "Brotli" + } else if entry.flags.contains(Flags::SNAPPY_COMPRESSED) { + "Snappy" + } else { + "None" + }; + + FileTableEntry { + id, + size: HumanBytes(entry.offset).to_string(), + flags: entry.flags, + compression: c_algo, + } + }) + .collect(); + + let mut table = Table::new(table_entries); + table + .with(Style::rounded()) + .with(Modify::list(Columns::new(..1), Alignment::left())); + + println!("{}", table); Ok(()) } @@ -117,5 +92,5 @@ struct FileTableEntry<'a> { id: &'a str, size: String, flags: Flags, - compression: String, + compression: &'static str, } diff --git a/vach-cli/src/commands/pack.rs b/vach-cli/src/commands/pack.rs index e0a5c3d3..086ba29b 100644 --- a/vach-cli/src/commands/pack.rs +++ b/vach-cli/src/commands/pack.rs @@ -30,8 +30,8 @@ impl Read for FileWrapper { }; let result = file.read(buf); - // Intercepts a file once it's finished reading to drop it, thus avoiding OS filesystem limitations easily - // Meaning we can safely drop the `fs::File` stored in this file wrapper + // Once the file is done reading, we drop the file handle + // TOo avoid hitting OS limitations if let Ok(0) = result { self.1.take(); }; @@ -107,7 +107,7 @@ impl CommandTrait for Evaluator { }; // Extract the inputs - let mut inputs: Vec = vec![]; + let mut inputs = vec![]; // Used to filter invalid inputs and excluded inputs let path_filter = |path: &PathBuf| match path.canonicalize() { @@ -188,8 +188,8 @@ impl CommandTrait for Evaluator { kp = Some(generated); } - let pbar = ProgressBar::new(inputs.len() as u64 + 5 + if truncate { 3 } else { 0 }); - pbar.set_style( + let progress = ProgressBar::new(inputs.len() as u64 + 5 + if truncate { 3 } else { 0 }); + progress.set_style( ProgressStyle::default_bar() .template(super::PROGRESS_BAR_STYLE)? .progress_chars("█░-") @@ -198,8 +198,8 @@ impl CommandTrait for Evaluator { // Since it wraps it's internal state in an arc, we can safely clone and send across threads let callback = |leaf: &Leaf, _: &RegistryEntry| { - pbar.inc(1); - pbar.set_message(leaf.id.clone()) + progress.inc(1); + progress.set_message(leaf.id.clone()) }; // Build a builder-config using the above extracted data @@ -234,42 +234,35 @@ impl CommandTrait for Evaluator { // Process the files for wrapper in &mut inputs { if !wrapper.0.exists() { - pbar.println(format!("Skipping {}, does not exist!", wrapper.0.to_string_lossy())); - - pbar.inc(1); + println!("Skipping {}, does not exist!", wrapper.0.to_string_lossy()); + progress.inc(1); continue; } - let id = wrapper - .0 - .to_string_lossy() - .trim_start_matches("./") - .trim_start_matches(".\\") - .to_string(); - println!("Preparing {} for packaging", id); + let id = wrapper.0.to_string_lossy().into_owned(); builder.add(wrapper, &id)?; } // Inform of success in input queue - pbar.inc(2); + progress.inc(2); builder.dump(output_file, &builder_config)?; - pbar.println(format!("Generated a new archive @ {}", output_path)); + progress.println(format!("Generated a new archive @ {}", output_path)); drop(builder); // Truncate original files if truncate { for wrapper in inputs { std::fs::remove_file(&wrapper.0)?; - pbar.finish(); - pbar.println(format!("Truncated original file @ {}", wrapper.0.to_string_lossy())); + progress.println(format!("Truncated original file @ {}", wrapper.0.to_string_lossy())); } - pbar.inc(3); + progress.inc(3); }; - pbar.inc(3); + progress.inc(3); + progress.finish(); Ok(()) } From 2fd26c8f2d004321f124e2349d1546cc43cf4065 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Sat, 30 Mar 2024 15:50:17 +0300 Subject: [PATCH 02/17] meh --- vach/src/global/header.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vach/src/global/header.rs b/vach/src/global/header.rs index c8973ca1..4de50d80 100644 --- a/vach/src/global/header.rs +++ b/vach/src/global/header.rs @@ -92,10 +92,10 @@ impl fmt::Display for ArchiveConfig { "[ArchiveConfig] magic: {}, has_public_key: {}", match str::from_utf8(&self.magic) { Ok(magic) => { - magic + magic.to_string() }, Err(_) => { - return fmt::Result::Err(fmt::Error); + format!("{:?}", &self.magic) }, }, has_pk @@ -166,9 +166,7 @@ impl Header { /// ### Errors /// - `io` errors pub(crate) fn from_handle(mut handle: T) -> InternalResult
{ - #![allow(clippy::uninit_assumed_init)] let mut buffer: [u8; Header::BASE_SIZE] = [0u8; Header::BASE_SIZE]; - handle.read_exact(&mut buffer)?; // Construct header From 83d4cf03d3ab2ad8cb21fc31a3c7a438cb1f5f9e Mon Sep 17 00:00:00 2001 From: sokorototo Date: Fri, 3 May 2024 17:31:58 +0300 Subject: [PATCH 03/17] small refactor --- vach/src/loader/archive.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/vach/src/loader/archive.rs b/vach/src/loader/archive.rs index 104f1929..f4bc04df 100644 --- a/vach/src/loader/archive.rs +++ b/vach/src/loader/archive.rs @@ -248,28 +248,28 @@ where } } -/// Given a data source and a [`RegistryEntry`], gets the adjacent raw data -pub(crate) fn read_entry(handle: &mut T, entry: &RegistryEntry) -> InternalResult> { - let mut buffer = Vec::with_capacity(entry.offset as usize + 64); - handle.seek(SeekFrom::Start(entry.location))?; - - let mut take = handle.take(entry.offset); - take.read_to_end(&mut buffer)?; - - Ok(buffer) -} - impl Archive where T: Read + Seek, { + /// Given a data source and a [`RegistryEntry`], gets the adjacent raw data + pub(crate) fn read_raw(handle: &mut T, entry: &RegistryEntry) -> InternalResult> { + let mut buffer = Vec::with_capacity(entry.offset as usize + 64); + handle.seek(SeekFrom::Start(entry.location))?; + + let mut take = handle.take(entry.offset); + take.read_to_end(&mut buffer)?; + + Ok(buffer) + } + /// Cheaper alternative to `fetch` that works best for single threaded applications. /// It does not lock the underlying [Mutex], since it requires a mutable reference. /// Therefore the borrow checker statically guarantees the operation is safe. Refer to [`Mutex::get_mut`](Mutex). pub fn fetch_mut(&mut self, id: impl AsRef) -> InternalResult { // The reason for this function's unnecessary complexity is it uses the provided functions independently, thus preventing an unnecessary allocation [MAYBE TOO MUCH?] if let Some(entry) = self.fetch_entry(&id) { - let raw = read_entry(self.handle.get_mut(), &entry)?; + let raw = Archive::read_raw(self.handle.get_mut(), &entry)?; // Prepare contextual variables // Decompress and|or decrypt the data @@ -293,7 +293,7 @@ where if let Some(entry) = self.fetch_entry(&id) { let raw = { let mut guard = self.handle.lock(); - read_entry(guard.by_ref(), &entry)? + Archive::read_raw(guard.by_ref(), &entry)? }; // Prepare contextual variables From 46fb18dca932ed4bbb8c53de73c1e94c88aa1f22 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Sat, 4 May 2024 07:53:42 +0300 Subject: [PATCH 04/17] ... --- vach/src/global/reg_entry.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vach/src/global/reg_entry.rs b/vach/src/global/reg_entry.rs index 9ecb7d8b..e4b45e01 100644 --- a/vach/src/global/reg_entry.rs +++ b/vach/src/global/reg_entry.rs @@ -11,14 +11,14 @@ pub struct RegistryEntry { pub flags: Flags, /// The content version of the extracted archive entry pub content_version: u8, - /// The signature of the data in the archive, used when verifying data authenticity - #[cfg(feature = "crypto")] - #[cfg_attr(docsrs, doc(cfg(feature = "crypto")))] - pub signature: Option, /// The location of the file in the archive, as an offset of bytes from the beginning of the file pub location: u64, /// The offset|size of the [`Leaf`](crate::builder::Leaf), in bytes. This is the actual number of bytes in the leaf endpoint. But the size of the data may vary once processed, ie when decompressed pub offset: u64, + /// The signature of the data in the archive, used when verifying data authenticity + #[cfg(feature = "crypto")] + #[cfg_attr(docsrs, doc(cfg(feature = "crypto")))] + pub signature: Option, } impl RegistryEntry { From e5c337dd556437cb44f32293db85224a1a4b820d Mon Sep 17 00:00:00 2001 From: sokorototo Date: Sat, 4 May 2024 07:57:42 +0300 Subject: [PATCH 05/17] ... --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1b9e679..b8778704 100644 --- a/README.md +++ b/README.md @@ -196,8 +196,8 @@ assert_eq!(archive.fetch_mut("d3")?.data.as_slice(), data_3); - [x] Data encryption. - [x] Benchmarks. - [x] Features to turn off (or to turn on) either the `Builder` or the `Loader` modules. -- [ ] Skynet, (coming _very_ soon). +- [x] `Some(examples)` instead of `None` +- [ ] Skynet, (coming _very_ soon). - [ ] Some proper benchmarking code. (Call for participation) -- [ ] `Some(examples)` directory instead of `None` > If you appreciate the works of this repo, consider dropping a star. It will be much appreciated; 🌟 From 9af2f95f923924800ad35c651c21c0435e8d48e3 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Sat, 4 May 2024 07:58:02 +0300 Subject: [PATCH 06/17] ... --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b8778704..373a2822 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ assert_eq!(archive.fetch_mut("d3")?.data.as_slice(), data_3); - [x] Benchmarks. - [x] Features to turn off (or to turn on) either the `Builder` or the `Loader` modules. - [x] `Some(examples)` instead of `None` -- [ ] Skynet, (coming _very_ soon). +- [ ] Skynet, (coming _very_ soon). - [ ] Some proper benchmarking code. (Call for participation) > If you appreciate the works of this repo, consider dropping a star. It will be much appreciated; 🌟 From 4fe012b6810e387b8bff8068a9c7edb39fe64ef2 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Tue, 14 May 2024 12:46:28 +0300 Subject: [PATCH 07/17] massively improved Builder architecture, altered crate structure --- .gitignore | 1 + Cargo.lock | 36 +- vach/Cargo.toml | 9 +- vach/src/crypto.rs | 2 +- vach/src/crypto_utils/mod.rs | 5 +- vach/src/global/compressor.rs | 4 +- vach/src/global/error.rs | 12 +- vach/src/global/flags.rs | 5 +- vach/src/global/header.rs | 2 +- vach/src/global/mod.rs | 5 +- vach/src/global/reg_entry.rs | 34 +- vach/src/global/result.rs | 4 - vach/src/lib.rs | 19 +- vach/src/loader/archive.rs | 50 ++- vach/src/tests/mod.rs | 3 +- vach/src/writer/builder/config.rs | 124 ------- vach/src/writer/builder/mod.rs | 350 ------------------- vach/src/writer/compress_mode.rs | 14 - vach/src/writer/config.rs | 38 ++- vach/src/writer/leaf.rs | 42 ++- vach/src/writer/mod.rs | 481 ++++++++++++++++++++++++++- vach/src/writer/prepared.rs | 7 + vach/test_data/encrypted/target.vach | Bin 191111 -> 191111 bytes vach/test_data/signed/target.vach | Bin 190972 -> 190972 bytes vach/test_data/simple/target.vach | Bin 190433 -> 190433 bytes 25 files changed, 638 insertions(+), 609 deletions(-) delete mode 100644 vach/src/writer/builder/config.rs delete mode 100644 vach/src/writer/builder/mod.rs delete mode 100644 vach/src/writer/compress_mode.rs create mode 100644 vach/src/writer/prepared.rs diff --git a/.gitignore b/.gitignore index 42aae610..ab636f52 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +vach-benchmarks/target .vscode .idea diff --git a/Cargo.lock b/Cargo.lock index 2efd7534..8a8f23ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,7 +131,18 @@ checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 2.5.1", +] + +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.0", ] [[package]] @@ -144,6 +155,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "brotli-decompressor" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -879,9 +900,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -889,9 +910,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -1219,10 +1240,9 @@ name = "vach" version = "0.5.5" dependencies = [ "aes-gcm", - "brotli", + "brotli 6.0.0", "ed25519-dalek", "lz4_flex", - "parking_lot", "rand", "rayon", "snap", @@ -1236,7 +1256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e36952a168221690ab9adc1dd46a7fb650449632f262d1fa86e4baed865eb7dc" dependencies = [ "aes-gcm", - "brotli", + "brotli 3.4.0", "ed25519-dalek", "lz4_flex", "parking_lot", diff --git a/vach/Cargo.toml b/vach/Cargo.toml index b092e852..a57cfb0f 100644 --- a/vach/Cargo.toml +++ b/vach/Cargo.toml @@ -33,11 +33,10 @@ lz4_flex = { version = "0.11.1", optional = true, default-features = false, feat "frame", ] } snap = { version = "1.1.1", optional = true } -brotli = { version = "3.4.0", optional = true } +brotli = { version = "6.0.0", optional = true } -# Multithreaded features -parking_lot = { version = "0.12.1" } -rayon = { version = "1.8.0", optional = true } +[dev-dependencies] +rayon = "1.10.0" [features] default = ["builder", "archive"] @@ -47,7 +46,7 @@ archive = [] builder = [] crypto = ["ed25519-dalek", "aes-gcm", "rand"] -multithreaded = ["rayon"] +multithreaded = [] compression = ["snap", "lz4_flex", "brotli"] [package.metadata.docs.rs] diff --git a/vach/src/crypto.rs b/vach/src/crypto.rs index 31a16246..906efc65 100644 --- a/vach/src/crypto.rs +++ b/vach/src/crypto.rs @@ -6,7 +6,7 @@ use aes_gcm::aead::Aead; use aes_gcm::aes::cipher::consts::U12; use aes_gcm::{Aes256Gcm, Nonce, KeyInit}; -pub use ed25519_dalek::{SigningKey, VerifyingKey, SecretKey, Signature}; +pub use ed25519_dalek::{SigningKey, VerifyingKey, Signature}; use crate::prelude::{InternalResult, InternalError}; diff --git a/vach/src/crypto_utils/mod.rs b/vach/src/crypto_utils/mod.rs index 94bfb668..54a9be16 100644 --- a/vach/src/crypto_utils/mod.rs +++ b/vach/src/crypto_utils/mod.rs @@ -3,10 +3,7 @@ use { rand::rngs::OsRng, - crate::{ - crypto, - global::{error::InternalError, result::InternalResult}, - }, + crate::{crypto, global::error::*}, std::io::Read, }; diff --git a/vach/src/global/compressor.rs b/vach/src/global/compressor.rs index 9356dedd..67e84a53 100644 --- a/vach/src/global/compressor.rs +++ b/vach/src/global/compressor.rs @@ -2,9 +2,9 @@ #![cfg_attr(docsrs, doc(cfg(feature = "compression")))] use std::io::{self, Read, Write}; -use crate::prelude::Flags; -use super::{error::InternalError, result::InternalResult}; +use crate::prelude::Flags; +use super::error::*; use lz4_flex as lz4; use snap; diff --git a/vach/src/global/error.rs b/vach/src/global/error.rs index 90270082..b673393b 100644 --- a/vach/src/global/error.rs +++ b/vach/src/global/error.rs @@ -1,6 +1,9 @@ -use std::{io, error}; +use std::{error, io, sync::Arc}; use thiserror::Error; +/// Internal `Result` type alias used by `vach`. Basically equal to: `Result` +pub type InternalResult = Result; + /// All errors manifestable within `vach` collected into a neat enum #[derive(Debug, Error)] pub enum InternalError { @@ -25,7 +28,7 @@ pub enum InternalError { MissingResourceError(String), /// Thrown when a leaf with an identical ID to a queued leaf is add with the `Builder::add(---)` functions #[error("[VachError::LeafAppendError] A leaf with the ID: {0} already exists. Consider changing the ID to prevent collisions")] - LeafAppendError(String), + LeafAppendError(Arc), /// Thrown when no `Keypair` is provided and an encrypted [Leaf](crate::builder::Leaf) is encountered #[error("[VachError::NoKeypairError] Unable to continue with cryptographic operation, as no keypair was supplied")] NoKeypairError, @@ -42,11 +45,8 @@ pub enum InternalError { /// An error that is thrown when the current archive attempts to load an incompatible version, contains the incompatible version #[error("The provided archive source has version: {}. While the current implementation has a spec-version: {}. The provided source is incompatible!", .0, crate::VERSION)] IncompatibleArchiveVersionError(u16), - /// An error that is thrown when if `Mutex` is poisoned, when a message doesn't go though an `mspc::sync_channel` or other sync related issues - #[error("[VachError::SyncError] {0}")] - SyncError(String), /// Errors thrown during compression or decompression #[error("[VachError::CompressorDecompressorError]: {0}")] #[cfg(feature = "compression")] - DeCompressionError(#[from] lz4_flex::frame::Error) + DeCompressionError(#[from] lz4_flex::frame::Error), } diff --git a/vach/src/global/flags.rs b/vach/src/global/flags.rs index 5281a595..f9855ec1 100644 --- a/vach/src/global/flags.rs +++ b/vach/src/global/flags.rs @@ -1,9 +1,10 @@ use std::fmt; -use super::{error::InternalError, result::InternalResult}; +use super::error::*; /// Abstracted flag access and manipulation `struct`. /// A knock-off minimal [bitflags](https://crates.io/crates/bitflags) of sorts. -#[derive(Copy, Clone, Default, PartialEq)] +#[derive(Copy, Clone, Default)] +#[repr(transparent)] pub struct Flags { pub(crate) bits: u32, } diff --git a/vach/src/global/header.rs b/vach/src/global/header.rs index 4de50d80..b971be38 100644 --- a/vach/src/global/header.rs +++ b/vach/src/global/header.rs @@ -2,7 +2,7 @@ use std::{fmt, io::Read, str}; #[cfg(feature = "crypto")] use crate::crypto; -use super::{error::InternalError, result::InternalResult, flags::Flags}; +use super::{error::*, flags::Flags}; /// Used to configure and give extra information to the [`Archive`](crate::archive::Archive) loader. /// Used exclusively in archive source and integrity validation. diff --git a/vach/src/global/mod.rs b/vach/src/global/mod.rs index eade1eee..7e82460d 100644 --- a/vach/src/global/mod.rs +++ b/vach/src/global/mod.rs @@ -1,7 +1,8 @@ // Globally available exports -pub mod compressor; pub mod error; + pub mod flags; pub mod header; pub mod reg_entry; -pub mod result; + +pub mod compressor; diff --git a/vach/src/global/reg_entry.rs b/vach/src/global/reg_entry.rs index e4b45e01..60e32aaf 100644 --- a/vach/src/global/reg_entry.rs +++ b/vach/src/global/reg_entry.rs @@ -1,5 +1,5 @@ -use std::{io::Read, fmt}; -use super::{result::InternalResult, flags::Flags}; +use std::{fmt, io::Read, sync::Arc}; +use super::{error::*, flags::Flags}; #[cfg(feature = "crypto")] use crate::crypto; @@ -7,6 +7,8 @@ use crate::crypto; /// Stand-alone meta-data for an archive entry(Leaf). This can be fetched without reading from the archive. #[derive(Debug, Clone)] pub struct RegistryEntry { + /// Self explanatory? + pub id: Arc, /// The flags extracted from the archive entry and parsed into a accessible struct pub flags: Flags, /// The content version of the extracted archive entry @@ -28,6 +30,7 @@ impl RegistryEntry { #[inline(always)] pub(crate) fn empty() -> RegistryEntry { RegistryEntry { + id: Arc::from(""), flags: Flags::empty(), content_version: 0, location: 0, @@ -41,7 +44,7 @@ impl RegistryEntry { /// Given a read handle, will proceed to read and parse bytes into a [`RegistryEntry`] struct. (de-serialization) /// ### Errors /// Produces `io` errors and if the bytes in the id section is not valid UTF-8 - pub(crate) fn from_handle(mut handle: T) -> InternalResult<(RegistryEntry, String)> { + pub(crate) fn from_handle(mut handle: T) -> InternalResult { let mut buffer: [u8; RegistryEntry::MIN_SIZE] = [0u8; RegistryEntry::MIN_SIZE]; handle.read_exact(&mut buffer)?; @@ -66,8 +69,6 @@ impl RegistryEntry { // If the `crypto` feature is turned off then the bytes are just read then discarded #[cfg(feature = "crypto")] { - use super::error::InternalError; - let sig = match crypto::Signature::try_from(sig_bytes) { Ok(sig) => sig, Err(err) => return Err(InternalError::ParseError(err.to_string())), @@ -83,6 +84,7 @@ impl RegistryEntry { // Build entry step manually, to prevent unnecessary `Default::default()` call, then changing fields individually let entry = RegistryEntry { + id: id.into(), flags, content_version, location, @@ -92,18 +94,27 @@ impl RegistryEntry { signature, }; - Ok((entry, id)) + Ok(entry) } /// Serializes a [`RegistryEntry`] struct into an array of bytes - pub(crate) fn bytes(&self, id_length: &u16) -> Vec { - let mut buffer = Vec::with_capacity(RegistryEntry::MIN_SIZE + 64); + pub(crate) fn bytes(&self) -> InternalResult> { + // Make sure the ID is not too big or else it will break the archive + let id = &self.id; + + if id.len() >= crate::MAX_ID_LENGTH { + let copy = id.to_string(); + return Err(InternalError::IDSizeOverflowError(copy)); + }; + + let mut buffer = Vec::with_capacity(RegistryEntry::MIN_SIZE + id.len()); + let len = id.len() as u16; buffer.extend_from_slice(&self.flags.bits().to_le_bytes()); buffer.extend_from_slice(&self.content_version.to_le_bytes()); buffer.extend_from_slice(&self.location.to_le_bytes()); buffer.extend_from_slice(&self.offset.to_le_bytes()); - buffer.extend_from_slice(&id_length.to_le_bytes()); + buffer.extend_from_slice(&len.to_le_bytes()); // Only write signature if one exists #[cfg(feature = "crypto")] @@ -111,7 +122,10 @@ impl RegistryEntry { buffer.extend_from_slice(&signature.to_bytes()) }; - buffer + // Append id + buffer.extend_from_slice(id.as_bytes()); + + Ok(buffer) } } diff --git a/vach/src/global/result.rs b/vach/src/global/result.rs index 91124069..e69de29b 100644 --- a/vach/src/global/result.rs +++ b/vach/src/global/result.rs @@ -1,4 +0,0 @@ -use super::error::InternalError; - -/// Internal `Result` type alias used by `vach`. Basically equal to: `Result` -pub type InternalResult = Result; diff --git a/vach/src/lib.rs b/vach/src/lib.rs index 4e0595b0..dae476b4 100644 --- a/vach/src/lib.rs +++ b/vach/src/lib.rs @@ -133,10 +133,6 @@ pub(crate) mod writer; #[cfg_attr(docsrs, doc(cfg(feature = "crypto")))] pub use rand; -#[cfg(feature = "multithreaded")] -#[cfg_attr(docsrs, doc(cfg(feature = "multithreaded")))] -pub use rayon; - /// Current [`vach`](crate) spec version. increments by ten with every spec change pub const VERSION: u16 = 30; @@ -160,9 +156,7 @@ pub const DEFAULT_MAGIC: &[u8; crate::MAGIC_LENGTH] = b"VfACH"; /// Consolidated import for crate logic; This module stores all `structs` associated with this crate. Constants can be accesses [directly](#constants) with `crate::` pub mod prelude { - pub use crate::global::{ - error::InternalError, result::InternalResult, flags::Flags, header::ArchiveConfig, reg_entry::RegistryEntry, - }; + pub use crate::global::{error::*, flags::Flags, header::ArchiveConfig, reg_entry::RegistryEntry}; #[cfg(feature = "crypto")] pub use crate::crypto::*; @@ -181,14 +175,9 @@ pub mod crypto; #[cfg(feature = "builder")] #[cfg_attr(docsrs, doc(cfg(feature = "builder")))] pub mod builder { - pub use crate::writer::{ - builder::{Builder, BuilderConfig}, - leaf::Leaf, - }; - pub use crate::global::{error::InternalError, result::InternalResult, flags::Flags}; + pub use crate::writer::*; + pub use crate::global::{error::*, flags::Flags}; - #[cfg(feature = "compression")] - pub use crate::writer::compress_mode::CompressMode; #[cfg(feature = "compression")] pub use crate::global::compressor::CompressionAlgorithm; } @@ -199,7 +188,7 @@ pub mod builder { pub mod archive { pub use crate::loader::{archive::Archive, resource::Resource}; pub use crate::global::{ - reg_entry::RegistryEntry, header::ArchiveConfig, error::InternalError, result::InternalResult, flags::Flags, + reg_entry::RegistryEntry, header::ArchiveConfig, error::*, flags::Flags, }; #[cfg(feature = "compression")] pub use crate::global::compressor::CompressionAlgorithm; diff --git a/vach/src/loader/archive.rs b/vach/src/loader/archive.rs index f4bc04df..34437c5f 100644 --- a/vach/src/loader/archive.rs +++ b/vach/src/loader/archive.rs @@ -1,20 +1,19 @@ use std::{ - str, - io::{Read, Seek, SeekFrom}, collections::HashMap, + io::{Read, Seek, SeekFrom}, + ops::DerefMut, + str, + sync::{Arc, Mutex}, }; use super::resource::Resource; use crate::global::{ - error::InternalError, + error::*, flags::Flags, header::{Header, ArchiveConfig}, reg_entry::RegistryEntry, - result::InternalResult, }; -use parking_lot::Mutex; - #[cfg(feature = "crypto")] use crate::crypto; @@ -33,9 +32,9 @@ pub struct Archive { /// Since all other work is done per thread handle: Mutex, - // Archive metadata + // Registry Data header: Header, - entries: HashMap, + entries: HashMap, RegistryEntry>, // Optional parts #[cfg(feature = "crypto")] @@ -67,7 +66,8 @@ impl std::fmt::Display for Archive { impl Archive { /// Consume the [Archive] and return the underlying handle - pub fn into_inner(self) -> T { + /// `None` if underlying + pub fn into_inner(self) -> Result> { self.handle.into_inner() } @@ -88,7 +88,8 @@ impl Archive { // If there is an error the data is flagged as invalid if let Some(signature) = entry.signature { - raw.extend_from_slice(id.as_bytes()); + let entry_bytes = entry.bytes()?; + raw.extend_from_slice(&entry_bytes); is_secure = pk.verify_strict(&raw, &signature).is_ok(); } @@ -189,26 +190,17 @@ where // Construct entries map for _ in 0..header.capacity { - let (entry, id) = RegistryEntry::from_handle(&mut handle)?; - entries.insert(id, entry); + let entry = RegistryEntry::from_handle(&mut handle)?; + entries.insert(entry.id.clone(), entry); } #[cfg(feature = "crypto")] { - // Build decryptor - let use_decryption = entries - .iter() - .any(|(_, entry)| entry.flags.contains(Flags::ENCRYPTED_FLAG)); - // Errors where no decryptor has been instantiated will be returned once a fetch is made to an encrypted resource - let decryptor = use_decryption - .then(|| { - config - .public_key - .as_ref() - .map(|pk| crypto::Encryptor::new(pk, config.magic)) - }) - .flatten(); + let decryptor = config + .public_key + .as_ref() + .map(|pk| crypto::Encryptor::new(pk, config.magic)); Ok(Archive { header, @@ -237,7 +229,7 @@ where /// Returns an immutable reference to the underlying [`HashMap`]. This hashmap stores [`RegistryEntry`] values and uses `String` keys. #[inline(always)] - pub fn entries(&self) -> &HashMap { + pub fn entries(&self) -> &HashMap, RegistryEntry> { &self.entries } @@ -269,7 +261,7 @@ where pub fn fetch_mut(&mut self, id: impl AsRef) -> InternalResult { // The reason for this function's unnecessary complexity is it uses the provided functions independently, thus preventing an unnecessary allocation [MAYBE TOO MUCH?] if let Some(entry) = self.fetch_entry(&id) { - let raw = Archive::read_raw(self.handle.get_mut(), &entry)?; + let raw = Archive::read_raw(self.handle.get_mut().unwrap(), &entry)?; // Prepare contextual variables // Decompress and|or decrypt the data @@ -292,8 +284,8 @@ where // The reason for this function's unnecessary complexity is it uses the provided functions independently, thus preventing an unnecessary allocation [MAYBE TOO MUCH?] if let Some(entry) = self.fetch_entry(&id) { let raw = { - let mut guard = self.handle.lock(); - Archive::read_raw(guard.by_ref(), &entry)? + let mut guard = self.handle.lock().unwrap(); + Archive::read_raw(guard.deref_mut(), &entry)? }; // Prepare contextual variables diff --git a/vach/src/tests/mod.rs b/vach/src/tests/mod.rs index f24460e0..9c609382 100644 --- a/vach/src/tests/mod.rs +++ b/vach/src/tests/mod.rs @@ -131,7 +131,7 @@ fn simple_fetch() -> InternalResult { fn builder_with_signature() -> InternalResult { let mut builder = Builder::default(); - let cb = |_: &Leaf, entry: &RegistryEntry| { + let cb = |entry: &RegistryEntry| { dbg!(entry); }; let mut build_config = BuilderConfig::default().callback(&cb); @@ -163,6 +163,7 @@ fn fetch_with_signature() -> InternalResult { config.load_public_key(keypair)?; let mut archive = Archive::with_config(target, &config)?; + dbg!(archive.entries()); let resource = archive.fetch_mut("test_data/song.txt")?; let song = str::from_utf8(&resource.data).unwrap(); diff --git a/vach/src/writer/builder/config.rs b/vach/src/writer/builder/config.rs deleted file mode 100644 index 19d19f83..00000000 --- a/vach/src/writer/builder/config.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::fmt::Debug; - -use crate::{ - global::{flags::Flags, reg_entry::RegistryEntry}, - builder::Leaf, -}; - -#[cfg(feature = "crypto")] -use crate::crypto; - -/// Allows for the customization of valid `vach` archives during their construction. -/// Such as custom `MAGIC`, custom `Header` flags and signing by providing a keypair. -pub struct BuilderConfig<'a> { - /// Used to write a unique magic sequence into the write target. - pub magic: [u8; crate::MAGIC_LENGTH], - /// Flags to be written into the `Header` section of the write target. - pub flags: Flags, - /// An optional keypair. If a key is provided, then the write target will have signatures for tamper verification. - #[cfg(feature = "crypto")] - #[cfg_attr(docsrs, doc(cfg(feature = "crypto")))] - pub keypair: Option, - /// An optional callback that is called every time a [Leaf](crate::builder::Leaf) finishes processing. - /// The callback get passed to it: a reference to the leaf and the generated registry entry. Use the RegEntry to get info on how the data was integrated for the given [`Leaf`]. - /// > **To avoid** the `implementation of "FnOnce" is not general enough` error consider adding types to the closure's parameters, as this is a type inference error. Rust somehow cannot infer enough information, [link](https://www.reddit.com/r/rust/comments/ntqu68/implementation_of_fnonce_is_not_general_enough/). - /// Usage: - /// ``` - /// use vach::prelude::{RegistryEntry, BuilderConfig, Leaf}; - /// - /// let builder_config = BuilderConfig::default(); - /// fn callback(leaf: &Leaf, reg_entry: &RegistryEntry) { - /// println!("Leaf: {leaf:?} has been processed into Entry: {reg_entry:?}") - /// } - /// - /// builder_config.callback(&callback); - /// ``` - #[allow(clippy::type_complexity)] - pub progress_callback: Option<&'a (dyn Fn(&Leaf, &RegistryEntry) + Send + Sync)>, -} - -impl<'a> Debug for BuilderConfig<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut f = f.debug_struct("BuilderConfig"); - - f.field("magic", &self.magic); - f.field("flags", &self.flags); - f.field( - "progress_callback", - if self.progress_callback.is_some() { - &"Some(&dyn Fn(id: &str, reg_entry: &RegistryEntry))" - } else { - &"None" - }, - ); - - #[cfg(feature = "crypto")] - f.field("keypair", &self.keypair); - - f.finish() - } -} - -impl<'a> BuilderConfig<'a> { - // Helper functions - /// Setter for the `keypair` field - #[cfg(feature = "crypto")] - pub fn keypair(mut self, keypair: crypto::SigningKey) -> Self { - self.keypair = Some(keypair); - self - } - - /// Setter for the `flags` field - ///``` - /// use vach::prelude::{Flags, BuilderConfig}; - /// - /// let config = BuilderConfig::default().flags(Flags::empty()); - ///``` - pub fn flags(mut self, flags: Flags) -> Self { - self.flags = flags; - self - } - - /// Setter for the `magic` field - ///``` - /// use vach::prelude::BuilderConfig; - /// let config = BuilderConfig::default().magic(*b"DbAfh"); - ///``` - pub fn magic(mut self, magic: [u8; 5]) -> BuilderConfig<'a> { - self.magic = magic; - self - } - - /// Setter for the `progress_callback` field - ///``` - /// use vach::prelude::{BuilderConfig, RegistryEntry, Leaf}; - /// - /// let callback = |_: &Leaf, entry: &RegistryEntry| { println!("Number of bytes written: {}", entry.offset) }; - /// let config = BuilderConfig::default().callback(&callback); - ///``` - pub fn callback(mut self, callback: &'a (dyn Fn(&Leaf, &RegistryEntry) + Send + Sync)) -> BuilderConfig<'a> { - self.progress_callback = Some(callback); - self - } - - // Keypair helpers - /// Parses and stores a keypair from a source. - /// ### Errors - /// If the call to `::utils::read_keypair()` fails to parse the data from the handle - #[cfg(feature = "crypto")] - pub fn load_keypair(&mut self, handle: T) -> crate::global::result::InternalResult { - crate::crypto_utils::read_keypair(handle).map(|kp| self.keypair = Some(kp)) - } -} - -impl<'a> Default for BuilderConfig<'a> { - fn default() -> BuilderConfig<'a> { - BuilderConfig { - flags: Flags::default(), - magic: *crate::DEFAULT_MAGIC, - progress_callback: None, - #[cfg(feature = "crypto")] - keypair: None, - } - } -} diff --git a/vach/src/writer/builder/mod.rs b/vach/src/writer/builder/mod.rs deleted file mode 100644 index f86d99cf..00000000 --- a/vach/src/writer/builder/mod.rs +++ /dev/null @@ -1,350 +0,0 @@ -use std::io::{BufWriter, Write, Seek, SeekFrom, Read}; -use std::collections::HashSet; -use std::path::Path; -use std::sync::atomic::{AtomicU64, AtomicUsize}; -use std::sync::{Arc, atomic::Ordering}; - -use parking_lot::Mutex; - -mod config; -pub use config::BuilderConfig; -use super::leaf::Leaf; - -#[cfg(feature = "compression")] -use {crate::global::compressor::Compressor, super::compress_mode::CompressMode}; - -use crate::global::error::InternalError; -use crate::global::result::InternalResult; -use crate::global::{header::Header, reg_entry::RegistryEntry, flags::Flags}; - -#[cfg(feature = "crypto")] -use {crate::crypto::Encryptor, ed25519_dalek::Signer}; - -/// The archive builder. Provides an interface with which one can configure and build valid `vach` archives. -#[derive(Default)] -pub struct Builder<'a> { - pub(crate) leafs: Vec>, - pub(crate) id_set: HashSet, - leaf_template: Leaf<'a>, -} - -impl<'a> Builder<'a> { - /// Instantiates a new [`Builder`] with an empty processing queue. - #[inline(always)] - pub fn new() -> Builder<'a> { - Builder::default() - } - - /// Appends a read handle wrapped in a [`Leaf`] into the processing queue. - /// The `data` is wrapped in the default [`Leaf`], without cloning the original data. - /// The second argument is the `ID` with which the embedded data will be tagged - /// ### Errors - /// - if a Leaf with the specified ID exists. - pub fn add(&mut self, data: D, id: impl AsRef) -> InternalResult { - let leaf = Leaf::new(data) - .id(id.as_ref().to_string()) - .template(&self.leaf_template); - - self.add_leaf(leaf) - } - - /// Removes all the [`Leaf`]s from the [`Builder`]. Leaves the `template` intact. Use this to re-use [`Builder`]s instead of instantiating new ones - pub fn clear(&mut self) { - self.id_set.clear(); - self.leafs.clear(); - } - - /// Loads all files from a directory, parses them into [`Leaf`]s and appends them into the processing queue. - /// An optional [`Leaf`] is passed as a template from which the new [`Leaf`]s shall implement, pass `None` to use the [`Builder`] internal default template. - /// Appended [`Leaf`]s have an `ID` in the form of of: `directory_name/file_name`. For example: `sounds/footstep.wav1, `sample/script.data` - /// ## Errors - /// - Any of the underlying calls to the filesystem fail. - /// - The internal call to `Builder::add_leaf()` fails. - pub fn add_dir(&mut self, path: impl AsRef, template: Option<&Leaf<'a>>) -> InternalResult { - use std::fs; - - let directory = fs::read_dir(path)?; - for file in directory { - let uri = file?.path(); - - let v = uri - .iter() - .map(|u| String::from(u.to_str().unwrap())) - .collect::>(); - - if !uri.is_dir() { - // Therefore a file - let file = fs::File::open(uri)?; - let leaf = Leaf::new(file) - .template(template.unwrap_or(&self.leaf_template)) - .id(&format!("{}/{}", v.get(v.len() - 2).unwrap(), v.last().unwrap())); - - self.add_leaf(leaf)?; - } - } - - Ok(()) - } - - /// Append a preconstructed [`Leaf`] into the processing queue. - /// [`Leaf`]s added directly do not implement data from the [`Builder`]s internal template. - /// ### Errors - /// - Returns an error if a [`Leaf`] with the specified `ID` exists. - pub fn add_leaf(&mut self, leaf: Leaf<'a>) -> InternalResult { - // Make sure no two leaves are written with the same ID - if !self.id_set.insert(leaf.id.clone()) { - Err(InternalError::LeafAppendError(leaf.id)) - } else { - self.leafs.push(leaf); - Ok(()) - } - } - - /// Avoid unnecessary boilerplate by auto-templating all [`Leaf`]s added with `Builder::add(--)` with the given template - /// ``` - /// use vach::builder::{Builder, Leaf}; - /// - /// let template = Leaf::default().version(12); - /// let mut builder = Builder::new().template(template); - /// - /// builder.add(b"JEB" as &[u8], "JEB_NAME").unwrap(); - /// // `JEB` is compressed and has a version of 12 - /// ``` - pub fn template(mut self, template: Leaf<'a>) -> Builder { - self.leaf_template = template; - self - } - - /// This iterates over all [`Leaf`]s in the processing queue, parses them and writes the bytes out into a the target. - /// Configure the custom *`MAGIC`*, `Header` flags and a [`Keypair`](crate::crypto::Keypair) using the [`BuilderConfig`] struct. - /// Wraps the `target` in [BufWriter]. Also calls `io::Seek` on the target, so no need for calling it externally for synchronization. - /// ### Errors - /// - Underlying `io` errors - /// - If the optional compression or compression stages fails - /// - If the requirements of a given stage, compression or encryption, are not met. Like not providing a keypair if a [`Leaf`] is to be encrypted. - pub fn dump(&mut self, mut target: W, config: &BuilderConfig) -> InternalResult { - // Keep track of how many bytes are written, and where bytes are being written - #[cfg(feature = "multithreaded")] - use rayon::prelude::*; - - #[allow(unused_mut)] - let mut reg_buffer_sync = Vec::new(); - - // Calculate the size of the registry and check for [`Leaf`]s that request for encryption - let leaf_offset_sync = { - self.leafs - .iter() - .map(|leaf| { - // The size of it's ID, the minimum size of an entry without a signature, and the size of a signature only if a signature is incorporated into the entry - leaf.id.len() + RegistryEntry::MIN_SIZE + { - #[cfg(feature = "crypto")] - if config.keypair.is_some() && leaf.sign { - crate::SIGNATURE_LENGTH - } else { - 0 - } - #[cfg(not(feature = "crypto"))] - { - 0 - } - } - }) - .reduce(|l1, l2| l1 + l2) - .unwrap_or(0) + Header::BASE_SIZE - } as u64; - - // Start at the very start of the file - target.seek(SeekFrom::Start(0))?; - - // Write header in order defined in the spec document - let mut wtr_sync = BufWriter::new(target); - wtr_sync.write_all(&config.magic)?; - - // INSERT flags - let mut temp = config.flags; - - #[cfg(feature = "crypto")] - if config.keypair.is_some() { - temp.force_set(Flags::SIGNED_FLAG, true); - }; - - wtr_sync.write_all(&temp.bits().to_le_bytes())?; - - // Write the version of the Archive Format|Builder|Loader - wtr_sync.write_all(&crate::VERSION.to_le_bytes())?; - wtr_sync.write_all(&(self.leafs.len() as u16).to_le_bytes())?; - - // Configure encryption - #[cfg(feature = "crypto")] - let use_encryption = self.leafs.iter().any(|leaf| leaf.encrypt); - - // Build encryptor - #[cfg(feature = "crypto")] - let encryptor = if use_encryption { - if let Some(keypair) = config.keypair.as_ref() { - Some(Encryptor::new(&keypair.verifying_key(), config.magic)) - } else { - return Err(InternalError::NoKeypairError); - } - } else { - None - }; - - // Define all arc-mutexes - let leaf_offset_arc = Arc::new(AtomicU64::new(leaf_offset_sync)); - let total_arc = Arc::new(AtomicUsize::new(Header::BASE_SIZE)); - let wtr_arc = Arc::new(Mutex::new(wtr_sync)); - let reg_buffer_arc = Arc::new(Mutex::new(reg_buffer_sync)); - - #[allow(unused_mut)] - let mut iter_mut; - - // Conditionally define iterator - #[cfg(feature = "multithreaded")] - { - iter_mut = self.leafs.as_mut_slice().par_iter_mut(); - } - - #[cfg(not(feature = "multithreaded"))] - { - iter_mut = self.leafs.iter_mut(); - } - - // Populate the archive glob - iter_mut.try_for_each(|leaf: &mut Leaf<'a>| -> InternalResult { - let mut entry: RegistryEntry = leaf.into(); - let mut raw = Vec::new(); - - // Compression comes first - #[cfg(feature = "compression")] - match leaf.compress { - CompressMode::Never => { - leaf.handle.read_to_end(&mut raw)?; - }, - CompressMode::Always => { - Compressor::new(&mut leaf.handle).compress(leaf.compression_algo, &mut raw)?; - - entry.flags.force_set(Flags::COMPRESSED_FLAG, true); - entry.flags.force_set(leaf.compression_algo.into(), true); - }, - CompressMode::Detect => { - let mut buffer = Vec::new(); - leaf.handle.read_to_end(&mut buffer)?; - - let mut compressed_data = Vec::new(); - Compressor::new(buffer.as_slice()).compress(leaf.compression_algo, &mut compressed_data)?; - - if compressed_data.len() <= buffer.len() { - entry.flags.force_set(Flags::COMPRESSED_FLAG, true); - entry.flags.force_set(leaf.compression_algo.into(), true); - - raw = compressed_data; - } else { - buffer.as_slice().read_to_end(&mut raw)?; - }; - }, - } - - // If the compression feature is turned off, simply reads into buffer - #[cfg(not(feature = "compression"))] - { - if entry.flags.contains(Flags::COMPRESSED_FLAG) { - return Err(InternalError::MissingFeatureError("compression")); - }; - - leaf.handle.read_to_end(&mut raw)?; - } - - // Encryption comes second - #[cfg(feature = "crypto")] - if leaf.encrypt { - if let Some(ex) = &encryptor { - raw = ex.encrypt(&raw)?; - - entry.flags.force_set(Flags::ENCRYPTED_FLAG, true); - } - } - - // Write processed leaf-contents and update offsets within `MutexGuard` protection - let glob_length = raw.len() as u64; - - { - // Lock writer - let mut wtr = wtr_arc.lock(); - - // Lock leaf_offset - let leaf_offset = leaf_offset_arc.load(Ordering::SeqCst); - - wtr.seek(SeekFrom::Start(leaf_offset))?; - wtr.write_all(&raw)?; - - // Update offset locations - entry.location = leaf_offset; - leaf_offset_arc.fetch_add(glob_length, Ordering::SeqCst); - - // Update number of bytes written - total_arc.fetch_add(glob_length as usize, Ordering::SeqCst); - }; - - // Update the offset of the entry to be the length of the glob - entry.offset = glob_length; - - #[cfg(feature = "crypto")] - if leaf.sign { - if let Some(keypair) = &config.keypair { - raw.extend_from_slice(leaf.id.as_bytes()); - - // The reason we include the path in the signature is to prevent mangling in the registry, - // For example, you may mangle the registry, causing this leaf to be addressed by a different reg_entry - // The path of that reg_entry + The data, when used to validate the signature, will produce an invalid signature. Invalidating the query - entry.signature = Some(keypair.sign(&raw)); - entry.flags.force_set(Flags::SIGNED_FLAG, true); - - // RAW has exhausted it's usefulness, we save memory by deallocating - drop(raw); - }; - } - - // Make sure the ID is not too big or else it will break the archive - if leaf.id.len() >= u16::MAX.into() { - let mut copy = leaf.id.clone(); - copy.truncate(25); - copy.shrink_to_fit(); - - return Err(InternalError::IDSizeOverflowError(copy)); - }; - - // Fetch bytes - let mut entry_bytes = entry.bytes(&(leaf.id.len() as u16)); - entry_bytes.extend_from_slice(leaf.id.as_bytes()); - - // Write to the registry-buffer and update total number of bytes written - { - let mut reg_buffer = reg_buffer_arc.lock(); - - reg_buffer.write_all(&entry_bytes)?; - total_arc.fetch_add(entry_bytes.len(), Ordering::SeqCst); - } - - // Call the progress callback bound within the [`BuilderConfig`] - if let Some(callback) = config.progress_callback { - callback(&leaf, &entry) - } - - Ok(()) - })?; - - // Write out the contents of the registry - { - let mut wtr = wtr_arc.lock(); - - let reg_buffer = reg_buffer_arc.lock(); - - wtr.seek(SeekFrom::Start(Header::BASE_SIZE as u64))?; - wtr.write_all(reg_buffer.as_slice())?; - }; - - // Return total number of bytes written - Ok(total_arc.load(Ordering::SeqCst)) - } -} diff --git a/vach/src/writer/compress_mode.rs b/vach/src/writer/compress_mode.rs deleted file mode 100644 index acead422..00000000 --- a/vach/src/writer/compress_mode.rs +++ /dev/null @@ -1,14 +0,0 @@ -/// Configures how `Leaf`s should be compressed. -/// Default is `CompressMode::Never`. -#[derive(Debug, Clone, Copy, Default)] -#[cfg(feature = "compression")] -#[cfg_attr(docsrs, doc(cfg(feature = "compression")))] -pub enum CompressMode { - /// The data is never compressed and is embedded as is. - #[default] - Never, - /// The data will always be compressed - Always, - /// The compressed data is used, only if it is smaller than the original data. - Detect, -} diff --git a/vach/src/writer/config.rs b/vach/src/writer/config.rs index 42f2873e..465f5e66 100644 --- a/vach/src/writer/config.rs +++ b/vach/src/writer/config.rs @@ -1,7 +1,6 @@ -use std::io; use std::fmt::Debug; -use crate::global::{flags::Flags, result::InternalResult, reg_entry::RegistryEntry}; +use crate::global::{flags::Flags, reg_entry::RegistryEntry}; #[cfg(feature = "crypto")] use crate::crypto; @@ -9,6 +8,9 @@ use crate::crypto; /// Allows for the customization of valid `vach` archives during their construction. /// Such as custom `MAGIC`, custom `Header` flags and signing by providing a keypair. pub struct BuilderConfig<'a> { + /// Number of threads to spawn during `Builder::dump`, defaults to 4 + #[cfg(feature = "multithreaded")] + pub num_threads: usize, /// Used to write a unique magic sequence into the write target. pub magic: [u8; crate::MAGIC_LENGTH], /// Flags to be written into the `Header` section of the write target. @@ -16,17 +18,22 @@ pub struct BuilderConfig<'a> { /// An optional keypair. If a key is provided, then the write target will have signatures for tamper verification. #[cfg(feature = "crypto")] #[cfg_attr(docsrs, doc(cfg(feature = "crypto")))] - pub keypair: Option, + pub keypair: Option, /// An optional callback that is called every time a [Leaf](crate::builder::Leaf) finishes processing. - /// The callback get passed to it: the leaf's id and the generated registry entry. Respectively. + /// The callback get passed to it: a reference to the leaf and the generated registry entry. Use the RegEntry to get info on how the data was integrated for the given [`Leaf`]. /// > **To avoid** the `implementation of "FnOnce" is not general enough` error consider adding types to the closure's parameters, as this is a type inference error. Rust somehow cannot infer enough information, [link](https://www.reddit.com/r/rust/comments/ntqu68/implementation_of_fnonce_is_not_general_enough/). /// Usage: /// ``` - /// use vach::builder::BuilderConfig; + /// use vach::prelude::{RegistryEntry, BuilderConfig, Leaf}; /// /// let builder_config = BuilderConfig::default(); + /// fn callback(leaf: &Leaf, reg_entry: &RegistryEntry) { + /// println!("Leaf: {leaf:?} has been processed into Entry: {reg_entry:?}") + /// } + /// + /// builder_config.callback(&callback); /// ``` - pub progress_callback: Option<&'a (dyn Fn(&str, &RegistryEntry) + Send + Sync)>, + pub progress_callback: Option<&'a dyn Fn(&RegistryEntry)>, } impl<'a> Debug for BuilderConfig<'a> { @@ -55,7 +62,7 @@ impl<'a> BuilderConfig<'a> { // Helper functions /// Setter for the `keypair` field #[cfg(feature = "crypto")] - pub fn keypair(mut self, keypair: crypto::Keypair) -> Self { + pub fn keypair(mut self, keypair: crypto::SigningKey) -> Self { self.keypair = Some(keypair); self } @@ -76,19 +83,19 @@ impl<'a> BuilderConfig<'a> { /// use vach::prelude::BuilderConfig; /// let config = BuilderConfig::default().magic(*b"DbAfh"); ///``` - pub fn magic(mut self, magic: [u8; crate::MAGIC_LENGTH]) -> BuilderConfig<'a> { + pub fn magic(mut self, magic: [u8; 5]) -> BuilderConfig<'a> { self.magic = magic; self } /// Setter for the `progress_callback` field ///``` - /// use vach::prelude::{BuilderConfig, RegistryEntry}; + /// use vach::prelude::{BuilderConfig, RegistryEntry, Leaf}; /// - /// let callback = |_: &str, entry: &RegistryEntry| { println!("Number of bytes written: {}", entry.offset) }; + /// let callback = |entry: &RegistryEntry| { println!("Number of bytes written: {}", entry.offset) }; /// let config = BuilderConfig::default().callback(&callback); ///``` - pub fn callback(mut self, callback: &'a (dyn Fn(&str, &RegistryEntry) + Send + Sync)) -> BuilderConfig<'a> { + pub fn callback(mut self, callback: &'a dyn Fn(&RegistryEntry)) -> BuilderConfig<'a> { self.progress_callback = Some(callback); self } @@ -96,17 +103,18 @@ impl<'a> BuilderConfig<'a> { // Keypair helpers /// Parses and stores a keypair from a source. /// ### Errors - /// If the call to `::crypto_utils::read_keypair()` fails to parse the data from the handle + /// If the call to `::utils::read_keypair()` fails to parse the data from the handle #[cfg(feature = "crypto")] - pub fn load_keypair(&mut self, handle: T) -> InternalResult { - self.keypair = Some(crate::crypto_utils::read_keypair(handle)?); - Ok(()) + pub fn load_keypair(&mut self, handle: T) -> crate::global::error::InternalResult { + crate::crypto_utils::read_keypair(handle).map(|kp| self.keypair = Some(kp)) } } impl<'a> Default for BuilderConfig<'a> { fn default() -> BuilderConfig<'a> { BuilderConfig { + #[cfg(feature = "multithreaded")] + num_threads: 4, flags: Flags::default(), magic: *crate::DEFAULT_MAGIC, progress_callback: None, diff --git a/vach/src/writer/leaf.rs b/vach/src/writer/leaf.rs index fb246ba1..2dba6d06 100644 --- a/vach/src/writer/leaf.rs +++ b/vach/src/writer/leaf.rs @@ -1,19 +1,33 @@ +#[cfg(feature = "compression")] +use crate::global::compressor::CompressionAlgorithm; use crate::global::{reg_entry::RegistryEntry, flags::Flags}; -#[cfg(feature = "compression")] -use {crate::global::compressor::CompressionAlgorithm, super::compress_mode::CompressMode}; +use std::{fmt, io::Read, sync::Arc}; -use std::{io::Read, fmt}; +/// Configures how `Leaf`s should be compressed. +/// Default is `CompressMode::Never`. +#[derive(Debug, Clone, Copy, Default)] +#[cfg(feature = "compression")] +#[cfg_attr(docsrs, doc(cfg(feature = "compression")))] +pub enum CompressMode { + /// The data is never compressed and is embedded as is. + #[default] + Never, + /// The data will always be compressed + Always, + /// The compressed data is used, only if it is smaller than the original data. + Detect, +} /// A wrapper around an [`io::Read`](std::io::Read) handle. /// Allows for multiple types of data implementing [`io::Read`](std::io::Read) to be used under one struct. /// Also used to configure how data will be processed and embedded into an write target. pub struct Leaf<'a> { /// The lifetime simply reflects to the [`Builder`](crate::builder::Builder)'s lifetime, meaning the handle must live longer than or the same as the Builder - pub(crate) handle: Box, + pub(crate) handle: Box, /// The `ID` under which the embedded data will be referenced - pub id: String, + pub id: Arc, /// The version of the content, allowing you to track obsolete data. pub content_version: u8, /// The flags that will go into the archive write target. @@ -50,7 +64,7 @@ impl<'a> Leaf<'a> { /// /// let leaf = Leaf::new(Cursor::new(vec![])); ///``` - pub fn new(handle: R) -> Leaf<'a> { + pub fn new(handle: R) -> Leaf<'a> { Leaf { handle: Box::new(handle), ..Default::default() @@ -111,8 +125,8 @@ impl<'a> Leaf<'a> { /// /// let leaf = Leaf::default().id("whatzitouya"); /// ``` - pub fn id(mut self, id: S) -> Self { - self.id = id.to_string(); + pub fn id>(mut self, id: S) -> Self { + self.id = Arc::from(id.as_ref()); self } @@ -164,7 +178,7 @@ impl<'a> Default for Leaf<'a> { Leaf { handle: Box::<&[u8]>::new(&[]), - id: Default::default(), + id: Arc::from(""), flags: Default::default(), content_version: Default::default(), @@ -207,9 +221,11 @@ impl<'a> fmt::Debug for Leaf<'a> { impl From<&mut Leaf<'_>> for RegistryEntry { fn from(leaf: &mut Leaf<'_>) -> Self { - let mut entry = RegistryEntry::empty(); - entry.content_version = leaf.content_version; - entry.flags = leaf.flags; - entry + RegistryEntry { + id: leaf.id.clone(), + flags: leaf.flags, + content_version: leaf.content_version, + ..RegistryEntry::empty() + } } } diff --git a/vach/src/writer/mod.rs b/vach/src/writer/mod.rs index 7ab0fb38..4375b81a 100644 --- a/vach/src/writer/mod.rs +++ b/vach/src/writer/mod.rs @@ -1,3 +1,478 @@ -pub mod builder; -pub mod compress_mode; -pub mod leaf; +use std::{ + io::{Write, Seek, SeekFrom, Read}, + collections::HashSet, + path::Path, + sync::Arc, +}; + +#[cfg(feature = "multithreaded")] +use std::{thread, sync::mpsc}; + +mod config; +mod leaf; +mod prepared; + +pub use config::BuilderConfig; +pub use leaf::Leaf; + +#[cfg(feature = "compression")] +pub use leaf::CompressMode; + +#[cfg(feature = "compression")] +use crate::global::compressor::Compressor; + +use crate::global::error::*; +use crate::global::{header::Header, reg_entry::RegistryEntry, flags::Flags}; + +#[cfg(feature = "crypto")] +use {crate::crypto::Encryptor, ed25519_dalek::Signer}; + +#[cfg(not(feature = "crypto"))] +type Encryptor = (); + +/// The archive builder. Provides an interface with which one can configure and build valid `vach` archives. +#[derive(Default)] +pub struct Builder<'a> { + pub(crate) leafs: Vec>, + pub(crate) id_set: HashSet>, + leaf_template: Leaf<'a>, +} + +impl<'a> Builder<'a> { + /// Instantiates a new [`Builder`] with an empty processing queue. + #[inline(always)] + pub fn new() -> Builder<'a> { + Builder::default() + } + + /// Appends a read handle wrapped in a [`Leaf`] into the processing queue. + /// The `data` is wrapped in the default [`Leaf`], without cloning the original data. + /// The second argument is the `ID` with which the embedded data will be tagged + /// ### Errors + /// - if a Leaf with the specified ID exists. + pub fn add(&mut self, data: D, id: impl AsRef) -> InternalResult { + let leaf = Leaf::new(data) + .id(id.as_ref().to_string()) + .template(&self.leaf_template); + + self.add_leaf(leaf) + } + + /// Removes all the [`Leaf`]s from the [`Builder`]. Leaves the `template` intact. Use this to re-use [`Builder`]s instead of instantiating new ones + pub fn clear(&mut self) { + self.id_set.clear(); + self.leafs.clear(); + } + + /// Loads all files from a directory, parses them into [`Leaf`]s and appends them into the processing queue. + /// An optional [`Leaf`] is passed as a template from which the new [`Leaf`]s shall implement, pass `None` to use the [`Builder`] internal default template. + /// Appended [`Leaf`]s have an `ID` in the form of of: `directory_name/file_name`. For example: `sounds/footstep.wav1, `sample/script.data` + /// ## Errors + /// - Any of the underlying calls to the filesystem fail. + /// - The internal call to `Builder::add_leaf()` fails. + pub fn add_dir(&mut self, path: impl AsRef, template: Option<&Leaf<'a>>) -> InternalResult { + use std::fs; + + let directory = fs::read_dir(path)?; + for file in directory { + let uri = file?.path(); + + let v = uri + .iter() + .map(|u| String::from(u.to_str().unwrap())) + .collect::>(); + + if !uri.is_dir() { + // Therefore a file + let file = fs::File::open(uri)?; + let leaf = Leaf::new(file) + .template(template.unwrap_or(&self.leaf_template)) + .id(&format!("{}/{}", v.get(v.len() - 2).unwrap(), v.last().unwrap())); + + self.add_leaf(leaf)?; + } + } + + Ok(()) + } + + /// Append a preconstructed [`Leaf`] into the processing queue. + /// [`Leaf`]s added directly do not implement data from the [`Builder`]s internal template. + /// ### Errors + /// - Returns an error if a [`Leaf`] with the specified `ID` exists. + pub fn add_leaf(&mut self, leaf: Leaf<'a>) -> InternalResult { + // Make sure no two leaves are written with the same ID + if !self.id_set.insert(leaf.id.clone()) { + Err(InternalError::LeafAppendError(leaf.id)) + } else { + self.leafs.push(leaf); + Ok(()) + } + } + + /// Avoid unnecessary boilerplate by auto-templating all [`Leaf`]s added with `Builder::add(--)` with the given template + /// ``` + /// use vach::builder::{Builder, Leaf}; + /// + /// let template = Leaf::default().version(12); + /// let mut builder = Builder::new().template(template); + /// + /// builder.add(b"JEB" as &[u8], "JEB_NAME").unwrap(); + /// // `JEB` is compressed and has a version of 12 + /// ``` + pub fn template(mut self, template: Leaf<'a>) -> Builder { + self.leaf_template = template; + self + } + + fn process_leaf(leaf: &mut Leaf<'a>, encryptor: Option<&Encryptor>) -> InternalResult { + let mut entry: RegistryEntry = leaf.into(); + let mut raw = Vec::new(); // 10MB + + // Compression comes first + #[cfg(feature = "compression")] + match leaf.compress { + CompressMode::Never => { + leaf.handle.read_to_end(&mut raw)?; + }, + CompressMode::Always => { + Compressor::new(&mut leaf.handle).compress(leaf.compression_algo, &mut raw)?; + + entry.flags.force_set(Flags::COMPRESSED_FLAG, true); + entry.flags.force_set(leaf.compression_algo.into(), true); + }, + CompressMode::Detect => { + let mut buffer = Vec::new(); + leaf.handle.read_to_end(&mut buffer)?; + + let mut compressed_data = Vec::new(); + Compressor::new(buffer.as_slice()).compress(leaf.compression_algo, &mut compressed_data)?; + + if compressed_data.len() <= buffer.len() { + entry.flags.force_set(Flags::COMPRESSED_FLAG, true); + entry.flags.force_set(leaf.compression_algo.into(), true); + + raw = compressed_data; + } else { + buffer.as_slice().read_to_end(&mut raw)?; + }; + }, + } + + // If the compression feature is turned off, simply reads into buffer + #[cfg(not(feature = "compression"))] + { + if entry.flags.contains(Flags::COMPRESSED_FLAG) { + return Err(InternalError::MissingFeatureError("compression")); + }; + + leaf.handle.read_to_end(&mut raw)?; + } + + // Encryption comes second + #[cfg(feature = "crypto")] + if leaf.encrypt { + if let Some(ex) = &encryptor { + raw = ex.encrypt(&raw)?; + entry.flags.force_set(Flags::ENCRYPTED_FLAG, true); + } + } + + Ok(prepared::Prepared { + data: raw, + entry, + #[cfg(feature = "crypto")] + sign: leaf.sign, + }) + } + + /// This iterates over all [`Leaf`]s in the processing queue, parses them and writes the bytes out into a the target. + /// Configure the custom *`MAGIC`*, `Header` flags and a [`Keypair`](crate::crypto::Keypair) using the [`BuilderConfig`] struct. + /// + /// ### Errors + /// - Underlying `io` errors + /// - If the optional compression or compression features aren't enabled + /// - If the requirements of a given stage, compression or encryption, are not met. Like not providing a keypair if a [`Leaf`] is to be encrypted. + pub fn dump(self, mut target: W, config: &BuilderConfig) -> InternalResult { + let Builder { mut leafs, .. } = self; + + // Calculate the size of the registry and check for [`Leaf`]s that request for encryption + let mut bytes_written = 0; + let mut leaf_offset = { + leafs + .iter() + .map(|leaf| { + // The size of it's ID, the minimum size of an entry without a signature, and the size of a signature only if a signature is incorporated into the entry + leaf.id.len() + RegistryEntry::MIN_SIZE + { + #[cfg(feature = "crypto")] + if config.keypair.is_some() && leaf.sign { + crate::SIGNATURE_LENGTH + } else { + 0 + } + #[cfg(not(feature = "crypto"))] + { + 0 + } + } + }) + .reduce(|l1, l2| l1 + l2) + .unwrap_or(0) + Header::BASE_SIZE + } as u64; + + // Start at the very start of the file + target.seek(SeekFrom::Start(0))?; + target.write_all(&config.magic)?; + + // INSERT flags + let mut temp = config.flags; + + #[cfg(feature = "crypto")] + if config.keypair.is_some() { + temp.force_set(Flags::SIGNED_FLAG, true); + }; + + // Write remaining Header + target.write_all(&temp.bits().to_le_bytes())?; + target.write_all(&crate::VERSION.to_le_bytes())?; + target.write_all(&(leafs.len() as u16).to_le_bytes())?; + + // Build encryptor + #[cfg(feature = "crypto")] + let encryptor = { + let use_encryption = leafs.iter().any(|leaf| leaf.encrypt); + if use_encryption { + if let Some(keypair) = config.keypair.as_ref() { + Some(Encryptor::new(&keypair.verifying_key(), config.magic)) + } else { + return Err(InternalError::NoKeypairError); + } + } else { + None + } + }; + + #[cfg(not(feature = "crypto"))] + let encryptor = None; + + // Callback for processing IO + let mut registry = Vec::with_capacity(leaf_offset as usize - Header::BASE_SIZE); + + #[allow(unused_mut)] + let mut write = |result: InternalResult| -> InternalResult<()> { + let mut result = result?; + let bytes = result.data.len() as u64; + + // write + target.seek(SeekFrom::Start(leaf_offset))?; + target.write_all(&result.data)?; + + // update entry + result.entry.location = leaf_offset; + result.entry.offset = bytes; + + // update state + leaf_offset += result.data.len() as u64; + bytes_written += bytes; + + // write out registry entry + #[cfg(feature = "crypto")] + if result.sign { + if let Some(keypair) = &config.keypair { + let entry_bytes = result.entry.bytes()?; + result.data.extend_from_slice(&entry_bytes); + + // Include registry data in the signature + result.entry.signature = Some(keypair.sign(&result.data)); + result.entry.flags.force_set(Flags::SIGNED_FLAG, true); + }; + } + + // write to registry buffer, this one might include the Signature + let entry_bytes = result.entry.bytes()?; + registry.write_all(&entry_bytes)?; + + // Call the progress callback bound within the [`BuilderConfig`] + config.progress_callback.inspect(|c| c(&result.entry)); + + Ok(()) + }; + + #[cfg(feature = "multithreaded")] + let (tx, rx) = mpsc::channel(); + + #[cfg(feature = "multithreaded")] + { + thread::scope(|s| -> InternalResult<()> { + let chunk_size = leafs.len() / config.num_threads; + let chunks = leafs.chunks_mut(chunk_size); + let encryptor = encryptor.as_ref(); + + // Spawn CPU threads + for chunk in chunks { + let queue = tx.clone(); + + s.spawn(move || { + for leaf in chunk { + let res = Builder::process_leaf(leaf, encryptor); + queue.send(res).unwrap(); + } + }); + } + + // Process IO, read results from + while let Ok(result) = rx.recv() { + write(result)? + } + + Ok(()) + })?; + }; + + #[cfg(not(feature = "multithreaded"))] + leafs + .iter_mut() + .map(|l| Builder::process_leaf(l, encryptor.as_ref())) + .try_for_each(write)?; + + // // Define all arc-mutexes + // let leaf_offset_arc = Arc::new(AtomicU64::new(leaf_offset)); + // let total_arc = Arc::new(AtomicUsize::new(Header::BASE_SIZE)); + // let wtr_arc = Arc::new(Mutex::new(target)); + // let reg_buffer_arc = Arc::new(Mutex::new(reg_buffer_sync)); + + // #[cfg(feature = "multithreaded")] + // let iter_mut = self.leafs.as_mut_slice().par_iter_mut(); + + // #[cfg(not(feature = "multithreaded"))] + // let iter_mut = self.leafs.iter_mut(); + + // // Populate the archive glob + // iter_mut.try_for_each(|leaf: &mut Leaf<'a>| -> InternalResult { + // let mut entry: RegistryEntry = leaf.into(); + // let mut raw = Vec::new(); + + // // Compression comes first + // #[cfg(feature = "compression")] + // match leaf.compress { + // CompressMode::Never => { + // leaf.handle.read_to_end(&mut raw)?; + // }, + // CompressMode::Always => { + // Compressor::new(&mut leaf.handle).compress(leaf.compression_algo, &mut raw)?; + + // entry.flags.force_set(Flags::COMPRESSED_FLAG, true); + // entry.flags.force_set(leaf.compression_algo.into(), true); + // }, + // CompressMode::Detect => { + // let mut buffer = Vec::new(); + // leaf.handle.read_to_end(&mut buffer)?; + + // let mut compressed_data = Vec::new(); + // Compressor::new(buffer.as_slice()).compress(leaf.compression_algo, &mut compressed_data)?; + + // if compressed_data.len() <= buffer.len() { + // entry.flags.force_set(Flags::COMPRESSED_FLAG, true); + // entry.flags.force_set(leaf.compression_algo.into(), true); + + // raw = compressed_data; + // } else { + // buffer.as_slice().read_to_end(&mut raw)?; + // }; + // }, + // } + + // // If the compression feature is turned off, simply reads into buffer + // #[cfg(not(feature = "compression"))] + // { + // if entry.flags.contains(Flags::COMPRESSED_FLAG) { + // return Err(InternalError::MissingFeatureError("compression")); + // }; + + // leaf.handle.read_to_end(&mut raw)?; + // } + + // // Encryption comes second + // #[cfg(feature = "crypto")] + // if leaf.encrypt { + // if let Some(ex) = &encryptor { + // raw = ex.encrypt(&raw)?; + + // entry.flags.force_set(Flags::ENCRYPTED_FLAG, true); + // } + // } + + // // Write processed leaf-contents and update offsets within `MutexGuard` protection + // let glob_length = raw.len() as u64; + + // { + // // Lock writer + // let mut wtr = wtr_arc.lock(); + + // // Lock leaf_offset + // let leaf_offset = leaf_offset_arc.load(Ordering::SeqCst); + + // wtr.seek(SeekFrom::Start(leaf_offset))?; + // wtr.write_all(&raw)?; + + // // Update offset locations + // entry.location = leaf_offset; + // leaf_offset_arc.fetch_add(glob_length, Ordering::SeqCst); + + // // Update number of bytes written + // total_arc.fetch_add(glob_length as usize, Ordering::SeqCst); + + // // Update the offset of the entry to be the length of the glob + // entry.offset = glob_length; + // }; + + // #[cfg(feature = "crypto")] + // if leaf.sign { + // if let Some(keypair) = &config.keypair { + // raw.extend_from_slice(leaf.id.as_bytes()); + + // // The reason we include the path in the signature is to prevent mangling in the registry, + // // For example, you may mangle the registry, causing this leaf to be addressed by a different reg_entry + // // The path of that reg_entry + The data, when used to validate the signature, will produce an invalid signature. Invalidating the query + // entry.signature = Some(keypair.sign(&raw)); + // entry.flags.force_set(Flags::SIGNED_FLAG, true); + + // // RAW has exhausted it's usefulness, we save memory by deallocating + // drop(raw); + // }; + // } + // // Fetch bytes + // let mut entry_bytes = entry.bytes(&(leaf.id.len() as u16))?; + // entry_bytes.extend_from_slice(leaf.id.as_bytes()); + + // // Write to the registry-buffer and update total number of bytes written + // { + // let mut reg_buffer = reg_buffer_arc.lock(); + + // reg_buffer.write_all(&entry_bytes)?; + // total_arc.fetch_add(entry_bytes.len(), Ordering::SeqCst); + // } + + // // Call the progress callback bound within the [`BuilderConfig`] + // if let Some(callback) = config.progress_callback { + // callback(&leaf, &entry) + // } + + // Ok(()) + // })?; + + // // Write out the contents of the registry + // { + // let mut wtr = wtr_arc.lock(); + + // let reg_buffer = reg_buffer_arc.lock(); + + // wtr.seek(SeekFrom::Start(Header::BASE_SIZE as u64))?; + // wtr.write_all(reg_buffer.as_slice())?; + // }; + + // // Return total number of bytes written + // Ok(total_arc.load(Ordering::SeqCst)) + Ok(bytes_written) + } +} diff --git a/vach/src/writer/prepared.rs b/vach/src/writer/prepared.rs new file mode 100644 index 00000000..3dff8f34 --- /dev/null +++ b/vach/src/writer/prepared.rs @@ -0,0 +1,7 @@ +// Unit of data ready to be inserted into a `Write + Clone` target during Building +pub(crate) struct Prepared { + pub(crate) data: Vec, + pub(crate) entry: super::RegistryEntry, + #[cfg(feature = "crypto")] + pub(crate) sign: bool, +} diff --git a/vach/test_data/encrypted/target.vach b/vach/test_data/encrypted/target.vach index c11e402903ab3f9dcc485e50286f1e8f3530d269..5b74ff83b5a8dec314f2418e1f402f8ff582db02 100644 GIT binary patch delta 733 zcmZp_%H4jIyFM(<(b delta 735 zcmZp_%H4jIyFM(<(bC%AL6&4PLaY*Eh!ra>H&=7{<>tR?>U{B% zCwjW)SpD|ZGw|dr+!<>VD!B91xx3r~OlMs#SGJoiK0W!Ww(a`Q`l|OQE&nBUOmZC` zl1=%kxq2lPC19I!Ss)gi1bdMI=%qRXp|yLq%WkdT`SMfn%e=s!cechIdy%*P=H?mu zgL7>Uh|gSUR&l@InnlE%ze@ei{VN-UrDgUWHWE0bXZP+vLpU44rsDj(bf`@e7|eJe zHmN`;20@16lFX9ijMQSi;yefg?4l^By7~(t5S1bf>rFh>SnbLpqb6nxoB1E#YqH~6 zyI;ro*E#M62i|=)?%r_Y_SJ*?PFk;t+f;Dw(86wqTCR(3H!kmq4%*du|9d?X!bM4` zsd~l9MVSQ<-)kj8-SiSd3o}gb6KnWtqpoMHSKrw3xqZT^CdN;>4L5D`4j%M<^*yrF z^vgrb*9W$EJJ)Mu+8Z2hTepGbVe>Y{r3y#o*?JpSgs>p&%E>Q63TQtsNK{%bfY1^Q zF2;Y>?76z=+ykMw+sm|?C!YOZ9@x&l+-X7jkM%DrBP=g`{%5i${D#g>^IsO?fyo>N zf96UzF7TY8Xi{KMe6yjh|i-*WW zLe&W|c*qxJ9(=eYxhkr7mdTdtHQH?();-$u=#nF!pi9_KqspKarYqCEWtiWqJl@^B zuhS##1+UUAp>^rkr0RPM_vKr2mZTP!#HS>dBM`kIub>|STA2+RKdMY!d@R`Qn@kNYFnJ$(H( zSI_p=?WZl_2TGSPAzPG~S)^A`Uz!BADK{Kq#ZG7-2r<0OzGu0s`xm>>%hKdy?AQ1Wea6h%kKS?|7kUCmFe$z38ly#f&Zq-vkyZjZMY7U&XL~yR&Y? z(Fs)-J?FkT_hi-O?{ljx^dw*1C_8&Hd&j-YZ9mzn zmk?T*K{);GpXjCUHVHg1XNiqBcXzCR5GDI%pV`3=mmUkQN@O3*twx|J~RKn zzIN}9r!`B2)ef(nxLoz*xo6fC zwNF5@s<1ROIlDN&s6?+ku{d}0TBZXb+npaV?PTA6(19_Mk%^0I^1~aJn^%NOzHFCr MV%#p}#H6_Z0L3B~%m4rY diff --git a/vach/test_data/simple/target.vach b/vach/test_data/simple/target.vach index 2214b3d1fa7907c47c849223edc71ba8bcbc885e..12617da59b859576ac3e6982b6ea1eab6ea71171 100644 GIT binary patch delta 244 zcmaEOp8Mf>?)tDaM`sTPAdq8V15tDa=49Hp@%T=5CN8eYg{798@5kMIvH1z-fe=BD k)SR4r9R=sa;^M@-l%hn%%?%tWjO{zz8Mp6nXVP2%0NiUE=l}o! delta 237 zcmaEOp8Mf>?)tDaM`sTPAdq8V15peN+aMH!Ae81{NH0oFEy>JF2MZjB3h!ixFjyFh z^YhXt@GuRAlasHb;G9@ooS2tVl&H9I+fMfFA#RK*jGLQ39thc- R&mr=%9jJHv4tFNa1pvf!FxCJ7 From 10feec4456d48f8a19fabc34d34ffbcb5fdcc3e6 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Tue, 14 May 2024 13:05:16 +0300 Subject: [PATCH 08/17] fixed small oversight --- vach/src/global/reg_entry.rs | 2 +- vach/src/tests/mod.rs | 15 +++------------ vach/src/writer/mod.rs | 4 ++++ vach/test_data/encrypted/target.vach | Bin 191111 -> 191111 bytes vach/test_data/signed/target.vach | Bin 190972 -> 190972 bytes vach/test_data/simple/target.vach | Bin 190433 -> 190433 bytes 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/vach/src/global/reg_entry.rs b/vach/src/global/reg_entry.rs index 60e32aaf..f5d7f644 100644 --- a/vach/src/global/reg_entry.rs +++ b/vach/src/global/reg_entry.rs @@ -30,7 +30,7 @@ impl RegistryEntry { #[inline(always)] pub(crate) fn empty() -> RegistryEntry { RegistryEntry { - id: Arc::from(""), + id: Arc::from(""), flags: Flags::empty(), content_version: 0, location: 0, diff --git a/vach/src/tests/mod.rs b/vach/src/tests/mod.rs index 9c609382..869118da 100644 --- a/vach/src/tests/mod.rs +++ b/vach/src/tests/mod.rs @@ -163,7 +163,6 @@ fn fetch_with_signature() -> InternalResult { config.load_public_key(keypair)?; let mut archive = Archive::with_config(target, &config)?; - dbg!(archive.entries()); let resource = archive.fetch_mut("test_data/song.txt")?; let song = str::from_utf8(&resource.data).unwrap(); @@ -178,17 +177,9 @@ fn fetch_with_signature() -> InternalResult { assert!(!not_signed_resource.authenticated); // Check authenticity of retrieved data + let song = song.trim(); println!("{}", song); - - // Windows bullshit - #[cfg(target_os = "windows")] - { - assert_eq!(song.len(), 2041); - } - #[cfg(not(any(target_os = "windows", target_os = "ios")))] - { - assert_eq!(song.len(), 1977); - } + assert_eq!(song.len(), 1977); assert!(resource.authenticated); assert!(resource.flags.contains(Flags::SIGNED_FLAG)); @@ -198,7 +189,7 @@ fn fetch_with_signature() -> InternalResult { #[test] #[cfg(feature = "crypto")] -fn edcryptor_test() -> InternalResult { +fn decryptor_test() -> InternalResult { use crate::crypto_utils::gen_keypair; let vk = gen_keypair().verifying_key(); diff --git a/vach/src/writer/mod.rs b/vach/src/writer/mod.rs index 4375b81a..64ac431f 100644 --- a/vach/src/writer/mod.rs +++ b/vach/src/writer/mod.rs @@ -335,6 +335,10 @@ impl<'a> Builder<'a> { .map(|l| Builder::process_leaf(l, encryptor.as_ref())) .try_for_each(write)?; + // write out Registry + target.seek(SeekFrom::Start(Header::BASE_SIZE as _))?; + target.write_all(®istry)?; + // // Define all arc-mutexes // let leaf_offset_arc = Arc::new(AtomicU64::new(leaf_offset)); // let total_arc = Arc::new(AtomicUsize::new(Header::BASE_SIZE)); diff --git a/vach/test_data/encrypted/target.vach b/vach/test_data/encrypted/target.vach index 5b74ff83b5a8dec314f2418e1f402f8ff582db02..c41c3b0680721bbad89de7caaaf4b849ebecee72 100644 GIT binary patch delta 717 zcmZp_%H4jIyFM(<(b_QLggO{i>O_U5jo-wN5skHKoGov0i=` zXGv;tNqkCTNuqvIYN}pwa#3bM3D9gVhL-Sph#e{~A+#_9w?WC*HtAHJz~;xrf!sAu z;x@70c;0U!Ep_O4q;A66+hS^3zm}huOm^pKTKW2o?$i4{tFjk{O1g9}U$V=xrM1;nZq4vb(c0w5MK2r;NO@Ap|-&sZ_fV}E3o@N}EykCLXBGrYCA!eg{` z+1f>q&fU3k^K@&F>qxule(uUFE3W-@sbkO{p9iUrlUoAL|aDd8%GaZ*TLS@7I#kW=N#Q zskO}cYqCi1@ov4x=JE^{2YAz#YaS0!;y`$+ARoz3iB1qJEEhm%2?jMww;NlR%D2}E zZ@aP3#F>*#zJKKd%bJ|~(sRmwZ=a&6b%^KHyJI46)a2B+eARC)omMaIm$h%<4xTw* zFS*XYc^S#3!qUv-?Be{Q620=o;#{!LW<7!=jg#P%!T|Ky7ir7l9d+AZb}{=GOL90? zEj;w!k6EB7-H|h6`T|B4+bhvOid}QPw{3rM$7{{koab}5xFrfHBv_m|tNP~#M=)cF zB|K>q=jWwEV`~CK@+*it<8 delta 717 zcmZp_%H4jIyFM(<(bAZ%&l!ucrz!lO2GELBD;ute|M*v?B9ur-`%d2`L2%Gs#djN?);BhVfWvCsM;4M z%TkhBToRv>Sdyrpl$xqnoLrPyPy#fYgCRG(9%{!+2rbMIzpo}RLh9#%hvzr{67-q< zO!(b{Rpc8<6bj4*~>6c(SaxwTzBSYZDiI)nm zPS#&*qjlMmsbT77qo4KuvGP-e&nr{_eR0tT*c?q}^o}{?WR>*S7so zo$I%nbjSWpH6J$XbD7IaUB3V4lEb7HXC8#73i6TsB;^FLLURFxmSEt!z4VIIqs{+* zW*yW!vg3VxwmZ-2i&B+JHYsbSp9{>@Srn~0{ZiA$Y5MPDCJF6N-05B~pF5wq(|q2Q ziieZ0)K)VfY$`0xOwKOOFDlV1Pb|&_`^@#IyANz82-G3CPHXzaWUlOm&6xmrst)mG_Pf9U(3X}eJvA{|8oG}zY2o@ delta 710 zcmex!nfuRW?)tDaM`sTP1_ll}26iAtZ%|Kf3(@QwwI3}&=*%@D-M;3vOzmr#7`Lxw IV)B0u0Cb}b9smFU diff --git a/vach/test_data/simple/target.vach b/vach/test_data/simple/target.vach index 12617da59b859576ac3e6982b6ea1eab6ea71171..09b3e202ed883196c94e946ca6be1e184b3bbd23 100644 GIT binary patch delta 53 zcmV-50LuU2&kNzt3$Skp5lCfhY;P9@U+0>D%eNM&qnZ!91~VRLg~Ze(&{A-8?z0%QS$ Ll<~Ke@c}8Igh&^H From 1873b9eba2b3d087b5a3a67f127c4839eb73bbf6 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Tue, 14 May 2024 15:00:23 +0300 Subject: [PATCH 09/17] fixed bug with RegistryEntry::encode --- vach-benchmarks/Cargo.toml | 7 ++++++- vach-benchmarks/benches/main.rs | 8 ++++---- vach/src/global/flags.rs | 2 +- vach/src/global/reg_entry.rs | 8 +++++--- vach/src/loader/archive.rs | 10 +++++----- vach/src/tests/mod.rs | 25 ++++++++++++++++--------- vach/src/writer/mod.rs | 8 ++++---- vach/test_data/simple/target.vach | Bin 190433 -> 190433 bytes 8 files changed, 41 insertions(+), 27 deletions(-) diff --git a/vach-benchmarks/Cargo.toml b/vach-benchmarks/Cargo.toml index 7bcfc2e9..1c037db6 100644 --- a/vach-benchmarks/Cargo.toml +++ b/vach-benchmarks/Cargo.toml @@ -7,7 +7,12 @@ publish = false [dependencies] criterion = "0.5.1" -vach = { path = "../vach", features = ["all"] } +vach = { path = "../vach", features = [ + "builder", + "archive", + "crypto", + "compression", +] } [[bench]] name = "benchmark" diff --git a/vach-benchmarks/benches/main.rs b/vach-benchmarks/benches/main.rs index bdb1b6c9..93df3370 100644 --- a/vach-benchmarks/benches/main.rs +++ b/vach-benchmarks/benches/main.rs @@ -42,8 +42,8 @@ pub fn criterion_benchmark(c: &mut Criterion) { let mut b_config = BuilderConfig::default().magic(*MAGIC); b_config.load_keypair(keypair_bytes).unwrap(); - let mut h_config = ArchiveConfig::default().magic(*MAGIC); - h_config.load_public_key(&keypair_bytes[32..]).unwrap(); + let mut a_config = ArchiveConfig::default().magic(*MAGIC); + a_config.load_public_key(&keypair_bytes[32..]).unwrap(); /* BUILDER BENCHMARKS */ let mut builder_group = c.benchmark_group("Builder"); @@ -98,7 +98,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { // Load data throughput_group.throughput(Throughput::Elements(3)); - let mut archive = Archive::with_config(&mut target, &h_config).unwrap(); + let mut archive = Archive::with_config(&mut target, &a_config).unwrap(); throughput_group.bench_function("Archive::fetch(---)", |b| { // Load data @@ -123,7 +123,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { c.bench_function("Archive::LOAD_NEW", |b| { // How fast it takes to load a new archive b.iter(|| { - black_box(Archive::with_config(&mut target, &h_config).unwrap()); + black_box(Archive::with_config(&mut target, &a_config).unwrap()); }) }); } diff --git a/vach/src/global/flags.rs b/vach/src/global/flags.rs index f9855ec1..50786c09 100644 --- a/vach/src/global/flags.rs +++ b/vach/src/global/flags.rs @@ -24,7 +24,7 @@ impl Flags { pub const LZ4_COMPRESSED: u32 = 0b_0100_0000_0000_0000_0000_0000_0000_0000; /// This entry was compressed using the [snappy](https://crates.io/crates/snap) scheme for balanced compression properties pub const SNAPPY_COMPRESSED: u32 = 0b_0010_0000_0000_0000_0000_0000_0000_0000; - /// This entry was compressed using the [brotli](https://crates.io/crates/brotli) scheme for higher compression ratios but *much* (depends on the quality of compression) slower compression speed + /// This entry was compressed using the [brotli](https://crates.io/crates/brotli) scheme for higher compression ratios but slower compression speed pub const BROTLI_COMPRESSED: u32 = 0b_0001_0000_0000_0000_0000_0000_0000_0000; /// The flag that denotes that the archive source has signatures diff --git a/vach/src/global/reg_entry.rs b/vach/src/global/reg_entry.rs index f5d7f644..f4c15e49 100644 --- a/vach/src/global/reg_entry.rs +++ b/vach/src/global/reg_entry.rs @@ -98,9 +98,9 @@ impl RegistryEntry { } /// Serializes a [`RegistryEntry`] struct into an array of bytes - pub(crate) fn bytes(&self) -> InternalResult> { + pub(crate) fn encode(&self, skip_signature: bool) -> InternalResult> { // Make sure the ID is not too big or else it will break the archive - let id = &self.id; + let id = self.id.as_ref(); if id.len() >= crate::MAX_ID_LENGTH { let copy = id.to_string(); @@ -119,7 +119,9 @@ impl RegistryEntry { // Only write signature if one exists #[cfg(feature = "crypto")] if let Some(signature) = self.signature { - buffer.extend_from_slice(&signature.to_bytes()) + if !skip_signature { + buffer.extend_from_slice(&signature.to_bytes()) + } }; // Append id diff --git a/vach/src/loader/archive.rs b/vach/src/loader/archive.rs index 34437c5f..30e1f9e4 100644 --- a/vach/src/loader/archive.rs +++ b/vach/src/loader/archive.rs @@ -84,16 +84,16 @@ impl Archive { // Validate signature only if a public key is passed with Some(PUBLIC_KEY) #[cfg(feature = "crypto")] if let Some(pk) = self.key { - let raw_size = raw.len(); - // If there is an error the data is flagged as invalid if let Some(signature) = entry.signature { - let entry_bytes = entry.bytes()?; + let raw_size = raw.len(); + + let entry_bytes = entry.encode(true)?; raw.extend_from_slice(&entry_bytes); + is_secure = pk.verify_strict(&raw, &signature).is_ok(); + raw.truncate(raw_size); } - - raw.truncate(raw_size); } // Add read layers diff --git a/vach/src/tests/mod.rs b/vach/src/tests/mod.rs index 869118da..615bbee9 100644 --- a/vach/src/tests/mod.rs +++ b/vach/src/tests/mod.rs @@ -110,7 +110,7 @@ fn builder_no_signature() -> InternalResult { #[test] #[cfg(all(feature = "compression", feature = "archive"))] -fn simple_fetch() -> InternalResult { +fn fetch_no_signature() -> InternalResult { let target = File::open(SIMPLE_TARGET)?; let mut archive = Archive::new(target)?; let resource = archive.fetch_mut("wasm")?; @@ -138,7 +138,8 @@ fn builder_with_signature() -> InternalResult { build_config.load_keypair(KEYPAIR.as_slice())?; - builder.add_dir("test_data", Some(&Leaf::default().sign(true)))?; + let template = Leaf::default().sign(true); + builder.add_dir("test_data", Some(&template))?; // Tests conditional signing builder.add_leaf(Leaf::default().id("not_signed").sign(false))?; @@ -233,7 +234,7 @@ fn builder_with_encryption() -> InternalResult { } #[test] -#[cfg(all(feature = "archive", feature = "crypto"))] +#[cfg(all(feature = "archive", feature = "crypto", feature = "compression"))] fn fetch_from_encrypted() -> InternalResult { let target = File::open(ENCRYPTED_TARGET)?; @@ -243,12 +244,18 @@ fn fetch_from_encrypted() -> InternalResult { config.load_public_key(public_key)?; let mut archive = Archive::with_config(target, &config)?; - let resource = archive.fetch_mut("test_data/quicksort.wasm")?; - assert_eq!(resource.data.len(), 106537); - assert!(resource.authenticated); - assert!(!resource.flags.contains(Flags::COMPRESSED_FLAG)); - assert!(resource.flags.contains(Flags::ENCRYPTED_FLAG)); + // read data + let not_signed = archive.fetch_mut("stitches.snitches")?; + let data = std::str::from_utf8(¬_signed.data).unwrap(); + assert_eq!(data, "Snitches get stitches, iOS sucks"); + + let signed = archive.fetch_mut("test_data/quicksort.wasm")?; + + assert_eq!(signed.data.len(), 106537); + assert!(signed.authenticated); + assert!(!signed.flags.contains(Flags::COMPRESSED_FLAG)); + assert!(signed.flags.contains(Flags::ENCRYPTED_FLAG)); Ok(()) } @@ -275,7 +282,7 @@ fn consolidated_example() -> InternalResult { let mut builder = Builder::new().template(Leaf::default().encrypt(true)); // Add data - let template = Leaf::default().encrypt(true).version(59); + let template = Leaf::default().encrypt(true).version(59).sign(true); builder.add_leaf(Leaf::new(data_1).id("d1").template(&template))?; builder.add_leaf(Leaf::new(data_2).id("d2").template(&template))?; builder.add_leaf(Leaf::new(data_3).id("d3").template(&template))?; diff --git a/vach/src/writer/mod.rs b/vach/src/writer/mod.rs index 64ac431f..66731ec4 100644 --- a/vach/src/writer/mod.rs +++ b/vach/src/writer/mod.rs @@ -172,7 +172,7 @@ impl<'a> Builder<'a> { // Encryption comes second #[cfg(feature = "crypto")] if leaf.encrypt { - if let Some(ex) = &encryptor { + if let Some(ex) = encryptor { raw = ex.encrypt(&raw)?; entry.flags.force_set(Flags::ENCRYPTED_FLAG, true); } @@ -279,8 +279,8 @@ impl<'a> Builder<'a> { #[cfg(feature = "crypto")] if result.sign { if let Some(keypair) = &config.keypair { - let entry_bytes = result.entry.bytes()?; - result.data.extend_from_slice(&entry_bytes); + let entry_bytes = result.entry.encode(true)?; + result.data.extend_from_slice(&entry_bytes); // DON"T FORGET TO TRUNCATE IF MODIFICATIONS ARE MADE // Include registry data in the signature result.entry.signature = Some(keypair.sign(&result.data)); @@ -289,7 +289,7 @@ impl<'a> Builder<'a> { } // write to registry buffer, this one might include the Signature - let entry_bytes = result.entry.bytes()?; + let entry_bytes = result.entry.encode(false)?; registry.write_all(&entry_bytes)?; // Call the progress callback bound within the [`BuilderConfig`] diff --git a/vach/test_data/simple/target.vach b/vach/test_data/simple/target.vach index 09b3e202ed883196c94e946ca6be1e184b3bbd23..749e4905b5daca076778f7c17329d7d5401840f2 100644 GIT binary patch delta 238 zcmaEOp8Mf>?)tDaM`sTPAdq8V15peN+aMIfPId@`g`qe0+% aF{LmHdZgy$Zl16-M`sTPAdq8Vn?BK#QDpi8Pe#^3M{L}-lU>jwH76%uN5MI Date: Tue, 21 May 2024 21:38:19 +0300 Subject: [PATCH 10/17] Removed redundant error documentation --- vach/src/crypto_utils/mod.rs | 10 ----- vach/src/global/flags.rs | 3 -- vach/src/global/header.rs | 8 ---- vach/src/global/reg_entry.rs | 4 +- vach/src/lib.rs | 4 +- vach/src/loader/archive.rs | 56 +++++++++++----------------- vach/src/{tests/mod.rs => tests.rs} | 22 +++++------ vach/src/writer/config.rs | 2 - vach/src/writer/mod.rs | 20 ++-------- vach/src/writer/prepared.rs | 14 +++---- vach/test_data/signed/target.vach | Bin 190972 -> 190720 bytes 11 files changed, 45 insertions(+), 98 deletions(-) rename vach/src/{tests/mod.rs => tests.rs} (96%) diff --git a/vach/src/crypto_utils/mod.rs b/vach/src/crypto_utils/mod.rs index 54a9be16..81c61824 100644 --- a/vach/src/crypto_utils/mod.rs +++ b/vach/src/crypto_utils/mod.rs @@ -22,8 +22,6 @@ pub fn gen_keypair() -> crypto::SigningKey { } /// Use this to read and parse a `Keypair` from a read stream -/// ### Errors -/// - If the data can't be parsed into a keypair pub fn read_keypair(mut handle: R) -> InternalResult { let mut keypair_bytes = [0; crate::SECRET_KEY_LENGTH + crate::PUBLIC_KEY_LENGTH]; handle.read_exact(&mut keypair_bytes)?; @@ -31,20 +29,12 @@ pub fn read_keypair(mut handle: R) -> InternalResult(mut handle: T) -> InternalResult { let mut keypair_bytes = [0; crate::PUBLIC_KEY_LENGTH]; handle.read_exact(&mut keypair_bytes)?; crypto::VerifyingKey::from_bytes(&keypair_bytes).map_err(|err| InternalError::ParseError(err.to_string())) } /// Read and parse a secret key from a read stream -/// -/// ### Errors -/// - If parsing of the secret key fails -/// - `io` errors pub fn read_secret_key(mut handle: T) -> InternalResult { let mut secret_bytes = [0; crate::SECRET_KEY_LENGTH]; handle.read_exact(&mut secret_bytes)?; diff --git a/vach/src/global/flags.rs b/vach/src/global/flags.rs index 50786c09..6df708bc 100644 --- a/vach/src/global/flags.rs +++ b/vach/src/global/flags.rs @@ -77,9 +77,6 @@ impl Flags { /// flag.set(0b0000_1000_0000_0001, false).unwrap(); // 0 flags remain zero /// assert_eq!(flag.bits(), 0b1000_0000_0000_0000); /// ``` - /// - /// ### Errors - /// - Trying to set a bit in the forbidden section of the flags pub fn set(&mut self, bit: u32, toggle: bool) -> InternalResult { if (Flags::RESERVED_MASK & bit) != 0 { return Err(InternalError::RestrictedFlagAccessError); diff --git a/vach/src/global/header.rs b/vach/src/global/header.rs index b971be38..00fc513a 100644 --- a/vach/src/global/header.rs +++ b/vach/src/global/header.rs @@ -50,10 +50,6 @@ impl ArchiveConfig { /// // config.load_public_key(&keypair_bytes).unwrap(); /// config.load_public_key(&keypair_bytes[32..]).unwrap(); /// ``` - /// - /// ### Errors - /// - If parsing of the public key fails - /// - `io` errors #[inline] #[cfg(feature = "crypto")] #[cfg_attr(docsrs, doc(cfg(feature = "crypto")))] @@ -147,8 +143,6 @@ impl Header { pub const CAPACITY_SIZE: usize = 2; /// Validates a `Header` with a template [ArchiveConfig] - /// ### Errors - /// - (in)validation of magic and archive version pub(crate) fn validate(config: &ArchiveConfig, header: &Header) -> InternalResult { // Validate magic if header.magic != config.magic { @@ -163,8 +157,6 @@ impl Header { Ok(()) } - /// ### Errors - /// - `io` errors pub(crate) fn from_handle(mut handle: T) -> InternalResult
{ let mut buffer: [u8; Header::BASE_SIZE] = [0u8; Header::BASE_SIZE]; handle.read_exact(&mut buffer)?; diff --git a/vach/src/global/reg_entry.rs b/vach/src/global/reg_entry.rs index f4c15e49..fbb6b219 100644 --- a/vach/src/global/reg_entry.rs +++ b/vach/src/global/reg_entry.rs @@ -42,8 +42,6 @@ impl RegistryEntry { } /// Given a read handle, will proceed to read and parse bytes into a [`RegistryEntry`] struct. (de-serialization) - /// ### Errors - /// Produces `io` errors and if the bytes in the id section is not valid UTF-8 pub(crate) fn from_handle(mut handle: T) -> InternalResult { let mut buffer: [u8; RegistryEntry::MIN_SIZE] = [0u8; RegistryEntry::MIN_SIZE]; handle.read_exact(&mut buffer)?; @@ -98,7 +96,7 @@ impl RegistryEntry { } /// Serializes a [`RegistryEntry`] struct into an array of bytes - pub(crate) fn encode(&self, skip_signature: bool) -> InternalResult> { + pub(crate) fn to_bytes(&self, skip_signature: bool) -> InternalResult> { // Make sure the ID is not too big or else it will break the archive let id = self.id.as_ref(); diff --git a/vach/src/lib.rs b/vach/src/lib.rs index dae476b4..e351f66d 100644 --- a/vach/src/lib.rs +++ b/vach/src/lib.rs @@ -187,9 +187,7 @@ pub mod builder { #[cfg_attr(docsrs, doc(cfg(feature = "archive")))] pub mod archive { pub use crate::loader::{archive::Archive, resource::Resource}; - pub use crate::global::{ - reg_entry::RegistryEntry, header::ArchiveConfig, error::*, flags::Flags, - }; + pub use crate::global::{reg_entry::RegistryEntry, header::ArchiveConfig, error::*, flags::Flags}; #[cfg(feature = "compression")] pub use crate::global::compressor::CompressionAlgorithm; } diff --git a/vach/src/loader/archive.rs b/vach/src/loader/archive.rs index 30e1f9e4..7374bf05 100644 --- a/vach/src/loader/archive.rs +++ b/vach/src/loader/archive.rs @@ -73,7 +73,7 @@ impl Archive { // Decompress and|or decrypt the data #[inline(never)] - fn process(&self, entry: &RegistryEntry, id: &str, mut raw: Vec) -> InternalResult<(Vec, bool)> { + fn process(&self, entry: &RegistryEntry, mut raw: Vec) -> InternalResult<(Vec, bool)> { /* Literally the hottest function in the block (🕶) */ // buffer_a originally contains the raw data @@ -88,7 +88,7 @@ impl Archive { if let Some(signature) = entry.signature { let raw_size = raw.len(); - let entry_bytes = entry.encode(true)?; + let entry_bytes = entry.to_bytes(true)?; raw.extend_from_slice(&entry_bytes); is_secure = pk.verify_strict(&raw, &signature).is_ok(); @@ -136,7 +136,11 @@ impl Archive { Compressor::new(source.as_slice()).decompress(CompressionAlgorithm::Snappy, &mut target)? } else { return InternalResult::Err(InternalError::OtherError( - format!("Unable to determine the compression algorithm used for entry with ID: {id}").into(), + format!( + "Unable to determine the compression algorithm used for entry: {}", + entry + ) + .into(), )); }; @@ -164,8 +168,6 @@ where /// ```skip /// Archive::with_config(HANDLE, &ArchiveConfig::default())?; /// ``` - /// ### Errors - /// - If the internal call to `Archive::with_config(-)` returns an error #[inline(always)] pub fn new(handle: T) -> InternalResult> { Archive::with_config(handle, &ArchiveConfig::default()) @@ -173,11 +175,6 @@ where /// Given a read handle, this will read and parse the data into an [`Archive`] struct. /// Pass a reference to [ArchiveConfig] and it will be used to validate the source and for further configuration. - /// ### Errors - /// - If parsing fails, an `Err(---)` is returned. - /// - The archive fails to validate - /// - `io` errors - /// - If any `ID`s are not valid UTF-8 pub fn with_config(mut handle: T, config: &ArchiveConfig) -> InternalResult> { // Start reading from the start of the input handle.seek(SeekFrom::Start(0))?; @@ -194,31 +191,20 @@ where entries.insert(entry.id.clone(), entry); } - #[cfg(feature = "crypto")] - { - // Errors where no decryptor has been instantiated will be returned once a fetch is made to an encrypted resource - let decryptor = config + let archive = Archive { + header, + handle: Mutex::new(handle), + entries, + + #[cfg(feature = "crypto")] + key: config.public_key, + #[cfg(feature = "crypto")] + decryptor: config .public_key .as_ref() - .map(|pk| crypto::Encryptor::new(pk, config.magic)); - - Ok(Archive { - header, - handle: Mutex::new(handle), - key: config.public_key, - entries, - decryptor, - }) - } - - #[cfg(not(feature = "crypto"))] - { - Ok(Archive { - header, - handle: Mutex::new(handle), - entries, - }) - } + .map(|pk| crypto::Encryptor::new(pk, config.magic)), + }; + Ok(archive) } /// Fetch a [`RegistryEntry`] from this [`Archive`]. @@ -265,7 +251,7 @@ where // Prepare contextual variables // Decompress and|or decrypt the data - let (buffer, is_secure) = self.process(&entry, id.as_ref(), raw)?; + let (buffer, is_secure) = self.process(&entry, raw)?; Ok(Resource { content_version: entry.content_version, @@ -290,7 +276,7 @@ where // Prepare contextual variables // Decompress and|or decrypt the data - let (buffer, is_secure) = self.process(&entry, id.as_ref(), raw)?; + let (buffer, is_secure) = self.process(&entry, raw)?; Ok(Resource { content_version: entry.content_version, diff --git a/vach/src/tests/mod.rs b/vach/src/tests.rs similarity index 96% rename from vach/src/tests/mod.rs rename to vach/src/tests.rs index 615bbee9..4fe55b74 100644 --- a/vach/src/tests/mod.rs +++ b/vach/src/tests.rs @@ -7,7 +7,7 @@ use crate::prelude::*; // Contains both the public key and secret key in the same file: // secret -> [u8; crate::SECRET_KEY_LENGTH], public -> [u8; crate::PUBLIC_KEY_LENGTH] -const KEYPAIR: &[u8; crate::SECRET_KEY_LENGTH + crate::PUBLIC_KEY_LENGTH] = include_bytes!("../../test_data/pair.pub"); +const KEYPAIR: &[u8; crate::SECRET_KEY_LENGTH + crate::PUBLIC_KEY_LENGTH] = include_bytes!("../test_data/pair.pub"); // The paths to the Archives, to be written|loaded const SIGNED_TARGET: &str = "test_data/signed/target.vach"; @@ -137,12 +137,15 @@ fn builder_with_signature() -> InternalResult { let mut build_config = BuilderConfig::default().callback(&cb); build_config.load_keypair(KEYPAIR.as_slice())?; + builder.add_dir("test_data", None)?; - let template = Leaf::default().sign(true); - builder.add_dir("test_data", Some(&template))?; + // sign and no sign! + builder.add_leaf(Leaf::default().id("not_signed"))?; - // Tests conditional signing - builder.add_leaf(Leaf::default().id("not_signed").sign(false))?; + let signed = Leaf::new(b"Don't forget to recite your beatitudes!" as &[u8]) + .id("signed") + .sign(true); + builder.add_leaf(signed)?; let mut target = File::create(SIGNED_TARGET)?; println!( @@ -169,19 +172,16 @@ fn fetch_with_signature() -> InternalResult { // The adjacent resource was flagged to not be signed let not_signed_resource = archive.fetch_mut("not_signed")?; - assert!(!not_signed_resource.flags.contains(Flags::SIGNED_FLAG)); - assert!(!not_signed_resource.authenticated); - - // The adjacent resource was flagged to not be signed - let not_signed_resource = archive.fetch_mut("not_signed")?; + dbg!(&archive.entries()); assert!(!not_signed_resource.flags.contains(Flags::SIGNED_FLAG)); assert!(!not_signed_resource.authenticated); // Check authenticity of retrieved data let song = song.trim(); - println!("{}", song); assert_eq!(song.len(), 1977); + let resource = archive.fetch_mut("signed")?; + assert!(resource.authenticated); assert!(resource.flags.contains(Flags::SIGNED_FLAG)); diff --git a/vach/src/writer/config.rs b/vach/src/writer/config.rs index 465f5e66..ad04a7a7 100644 --- a/vach/src/writer/config.rs +++ b/vach/src/writer/config.rs @@ -102,8 +102,6 @@ impl<'a> BuilderConfig<'a> { // Keypair helpers /// Parses and stores a keypair from a source. - /// ### Errors - /// If the call to `::utils::read_keypair()` fails to parse the data from the handle #[cfg(feature = "crypto")] pub fn load_keypair(&mut self, handle: T) -> crate::global::error::InternalResult { crate::crypto_utils::read_keypair(handle).map(|kp| self.keypair = Some(kp)) diff --git a/vach/src/writer/mod.rs b/vach/src/writer/mod.rs index 66731ec4..7d191535 100644 --- a/vach/src/writer/mod.rs +++ b/vach/src/writer/mod.rs @@ -48,8 +48,6 @@ impl<'a> Builder<'a> { /// Appends a read handle wrapped in a [`Leaf`] into the processing queue. /// The `data` is wrapped in the default [`Leaf`], without cloning the original data. /// The second argument is the `ID` with which the embedded data will be tagged - /// ### Errors - /// - if a Leaf with the specified ID exists. pub fn add(&mut self, data: D, id: impl AsRef) -> InternalResult { let leaf = Leaf::new(data) .id(id.as_ref().to_string()) @@ -67,9 +65,6 @@ impl<'a> Builder<'a> { /// Loads all files from a directory, parses them into [`Leaf`]s and appends them into the processing queue. /// An optional [`Leaf`] is passed as a template from which the new [`Leaf`]s shall implement, pass `None` to use the [`Builder`] internal default template. /// Appended [`Leaf`]s have an `ID` in the form of of: `directory_name/file_name`. For example: `sounds/footstep.wav1, `sample/script.data` - /// ## Errors - /// - Any of the underlying calls to the filesystem fail. - /// - The internal call to `Builder::add_leaf()` fails. pub fn add_dir(&mut self, path: impl AsRef, template: Option<&Leaf<'a>>) -> InternalResult { use std::fs; @@ -96,10 +91,8 @@ impl<'a> Builder<'a> { Ok(()) } - /// Append a preconstructed [`Leaf`] into the processing queue. - /// [`Leaf`]s added directly do not implement data from the [`Builder`]s internal template. - /// ### Errors - /// - Returns an error if a [`Leaf`] with the specified `ID` exists. + /// Directly add a [`Leaf`] to the [`Builder`] + /// [`Leaf`]s added directly do not inherit data from the [`Builder`]s template. pub fn add_leaf(&mut self, leaf: Leaf<'a>) -> InternalResult { // Make sure no two leaves are written with the same ID if !self.id_set.insert(leaf.id.clone()) { @@ -188,11 +181,6 @@ impl<'a> Builder<'a> { /// This iterates over all [`Leaf`]s in the processing queue, parses them and writes the bytes out into a the target. /// Configure the custom *`MAGIC`*, `Header` flags and a [`Keypair`](crate::crypto::Keypair) using the [`BuilderConfig`] struct. - /// - /// ### Errors - /// - Underlying `io` errors - /// - If the optional compression or compression features aren't enabled - /// - If the requirements of a given stage, compression or encryption, are not met. Like not providing a keypair if a [`Leaf`] is to be encrypted. pub fn dump(self, mut target: W, config: &BuilderConfig) -> InternalResult { let Builder { mut leafs, .. } = self; @@ -279,7 +267,7 @@ impl<'a> Builder<'a> { #[cfg(feature = "crypto")] if result.sign { if let Some(keypair) = &config.keypair { - let entry_bytes = result.entry.encode(true)?; + let entry_bytes = result.entry.to_bytes(true)?; result.data.extend_from_slice(&entry_bytes); // DON"T FORGET TO TRUNCATE IF MODIFICATIONS ARE MADE // Include registry data in the signature @@ -289,7 +277,7 @@ impl<'a> Builder<'a> { } // write to registry buffer, this one might include the Signature - let entry_bytes = result.entry.encode(false)?; + let entry_bytes = result.entry.to_bytes(false)?; registry.write_all(&entry_bytes)?; // Call the progress callback bound within the [`BuilderConfig`] diff --git a/vach/src/writer/prepared.rs b/vach/src/writer/prepared.rs index 3dff8f34..056d09b9 100644 --- a/vach/src/writer/prepared.rs +++ b/vach/src/writer/prepared.rs @@ -1,7 +1,7 @@ -// Unit of data ready to be inserted into a `Write + Clone` target during Building -pub(crate) struct Prepared { - pub(crate) data: Vec, - pub(crate) entry: super::RegistryEntry, - #[cfg(feature = "crypto")] - pub(crate) sign: bool, -} +// Unit of data ready to be inserted into a `Write + Clone` target during Building +pub(crate) struct Prepared { + pub(crate) data: Vec, + pub(crate) entry: super::RegistryEntry, + #[cfg(feature = "crypto")] + pub(crate) sign: bool, +} diff --git a/vach/test_data/signed/target.vach b/vach/test_data/signed/target.vach index 1c5eb60324a296ceb0152758583b808804daf635..15ba76f6fd13b48c8683edcbba98a24682b37025 100644 GIT binary patch delta 408 zcmex!nY-Z>*}2K!h2Dkc=ou%q-F?C`|%O&4Vh8gxZOsFh4a{ zucQKE({~4m3e5!&T4FL6lT1AWgTg~5ut7W7Ayy$9P@JEa4maTD3y4Cfix{{V^72dK zi!;;nQd59p91Ji8>QEJI3=XGOJ)C)9W}wE}wOkx;54a0CESWk-S-o_J;$P0N1w4{+ z^7ZWA27Xzg!UZ$=TPxq~I(v81@9-tw#u4eqJrq=Lvo$YgYG2O8xP3VjQ|NPbm;5~S n5{0z0>mRP76s zWhqH5E{RV`EJ@T)N=?-(PA(#Q~aapI-ItCRKD+Gt(2 zWNMhY+307zf9(Ba-_AB|(`!X@!s076SJ@ZDZ|Me5P!e?PMh z>K)nfK0e!>XZ1y?N+orhlr_`O1?K83idLO|scGXh{r54Gg!U)ybeGSa&)jJ~?@GnP z$yaKtC!Uf429@(8NPO=EM?VA52f51~uLsKV+N|FHJn-O_H}Tu~8FRV~q+P$ez5dxQ zwZlWi=FZi=V`rAlP0tQ_t-K}n7;B(xmiDDxR$X#}VY@1s5FRMb&r64TfPvx9OK8G^ a#t9cgUOrPvd~*w9dkZ7u_7+B_(B}Yu2>0>; From f1f8f5999efeb567d123c0bf41dc5e50f101f094 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Tue, 21 May 2024 21:41:46 +0300 Subject: [PATCH 11/17] prevent pushing empty vach files --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ab636f52..c75ae134 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,8 @@ perf.data perf.data.old workspace.code-workspace .cargo/config.toml -**.vach + +# Test Data +vach/test_data/encrypted +vach/test_data/signed +vach/test_data/simple From 8290407739675016f212562ace9bf611db294b76 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Wed, 22 May 2024 10:35:44 +0300 Subject: [PATCH 12/17] fixed Builder bug --- vach/src/global/flags.rs | 2 +- vach/src/tests.rs | 1 - vach/src/writer/mod.rs | 143 +-------------------------- vach/test_data/encrypted/target.vach | Bin 191111 -> 191111 bytes vach/test_data/signed/target.vach | Bin 190720 -> 190720 bytes 5 files changed, 4 insertions(+), 142 deletions(-) diff --git a/vach/src/global/flags.rs b/vach/src/global/flags.rs index 6df708bc..0646091d 100644 --- a/vach/src/global/flags.rs +++ b/vach/src/global/flags.rs @@ -129,7 +129,7 @@ impl fmt::Debug for Flags { write!( f, - "Flags[{}{}{}]: <{}u16 : {:#016b}>", + "Flags[{}{}{}]: <{}u32 : {:#032b}>", compressed, encrypted, signed, self.bits, self.bits ) } diff --git a/vach/src/tests.rs b/vach/src/tests.rs index 4fe55b74..bb482ed5 100644 --- a/vach/src/tests.rs +++ b/vach/src/tests.rs @@ -172,7 +172,6 @@ fn fetch_with_signature() -> InternalResult { // The adjacent resource was flagged to not be signed let not_signed_resource = archive.fetch_mut("not_signed")?; - dbg!(&archive.entries()); assert!(!not_signed_resource.flags.contains(Flags::SIGNED_FLAG)); assert!(!not_signed_resource.authenticated); diff --git a/vach/src/writer/mod.rs b/vach/src/writer/mod.rs index 7d191535..57727242 100644 --- a/vach/src/writer/mod.rs +++ b/vach/src/writer/mod.rs @@ -267,12 +267,13 @@ impl<'a> Builder<'a> { #[cfg(feature = "crypto")] if result.sign { if let Some(keypair) = &config.keypair { + result.entry.flags.force_set(Flags::SIGNED_FLAG, true); + let entry_bytes = result.entry.to_bytes(true)?; - result.data.extend_from_slice(&entry_bytes); // DON"T FORGET TO TRUNCATE IF MODIFICATIONS ARE MADE + result.data.extend_from_slice(&entry_bytes); // Include registry data in the signature result.entry.signature = Some(keypair.sign(&result.data)); - result.entry.flags.force_set(Flags::SIGNED_FLAG, true); }; } @@ -327,144 +328,6 @@ impl<'a> Builder<'a> { target.seek(SeekFrom::Start(Header::BASE_SIZE as _))?; target.write_all(®istry)?; - // // Define all arc-mutexes - // let leaf_offset_arc = Arc::new(AtomicU64::new(leaf_offset)); - // let total_arc = Arc::new(AtomicUsize::new(Header::BASE_SIZE)); - // let wtr_arc = Arc::new(Mutex::new(target)); - // let reg_buffer_arc = Arc::new(Mutex::new(reg_buffer_sync)); - - // #[cfg(feature = "multithreaded")] - // let iter_mut = self.leafs.as_mut_slice().par_iter_mut(); - - // #[cfg(not(feature = "multithreaded"))] - // let iter_mut = self.leafs.iter_mut(); - - // // Populate the archive glob - // iter_mut.try_for_each(|leaf: &mut Leaf<'a>| -> InternalResult { - // let mut entry: RegistryEntry = leaf.into(); - // let mut raw = Vec::new(); - - // // Compression comes first - // #[cfg(feature = "compression")] - // match leaf.compress { - // CompressMode::Never => { - // leaf.handle.read_to_end(&mut raw)?; - // }, - // CompressMode::Always => { - // Compressor::new(&mut leaf.handle).compress(leaf.compression_algo, &mut raw)?; - - // entry.flags.force_set(Flags::COMPRESSED_FLAG, true); - // entry.flags.force_set(leaf.compression_algo.into(), true); - // }, - // CompressMode::Detect => { - // let mut buffer = Vec::new(); - // leaf.handle.read_to_end(&mut buffer)?; - - // let mut compressed_data = Vec::new(); - // Compressor::new(buffer.as_slice()).compress(leaf.compression_algo, &mut compressed_data)?; - - // if compressed_data.len() <= buffer.len() { - // entry.flags.force_set(Flags::COMPRESSED_FLAG, true); - // entry.flags.force_set(leaf.compression_algo.into(), true); - - // raw = compressed_data; - // } else { - // buffer.as_slice().read_to_end(&mut raw)?; - // }; - // }, - // } - - // // If the compression feature is turned off, simply reads into buffer - // #[cfg(not(feature = "compression"))] - // { - // if entry.flags.contains(Flags::COMPRESSED_FLAG) { - // return Err(InternalError::MissingFeatureError("compression")); - // }; - - // leaf.handle.read_to_end(&mut raw)?; - // } - - // // Encryption comes second - // #[cfg(feature = "crypto")] - // if leaf.encrypt { - // if let Some(ex) = &encryptor { - // raw = ex.encrypt(&raw)?; - - // entry.flags.force_set(Flags::ENCRYPTED_FLAG, true); - // } - // } - - // // Write processed leaf-contents and update offsets within `MutexGuard` protection - // let glob_length = raw.len() as u64; - - // { - // // Lock writer - // let mut wtr = wtr_arc.lock(); - - // // Lock leaf_offset - // let leaf_offset = leaf_offset_arc.load(Ordering::SeqCst); - - // wtr.seek(SeekFrom::Start(leaf_offset))?; - // wtr.write_all(&raw)?; - - // // Update offset locations - // entry.location = leaf_offset; - // leaf_offset_arc.fetch_add(glob_length, Ordering::SeqCst); - - // // Update number of bytes written - // total_arc.fetch_add(glob_length as usize, Ordering::SeqCst); - - // // Update the offset of the entry to be the length of the glob - // entry.offset = glob_length; - // }; - - // #[cfg(feature = "crypto")] - // if leaf.sign { - // if let Some(keypair) = &config.keypair { - // raw.extend_from_slice(leaf.id.as_bytes()); - - // // The reason we include the path in the signature is to prevent mangling in the registry, - // // For example, you may mangle the registry, causing this leaf to be addressed by a different reg_entry - // // The path of that reg_entry + The data, when used to validate the signature, will produce an invalid signature. Invalidating the query - // entry.signature = Some(keypair.sign(&raw)); - // entry.flags.force_set(Flags::SIGNED_FLAG, true); - - // // RAW has exhausted it's usefulness, we save memory by deallocating - // drop(raw); - // }; - // } - // // Fetch bytes - // let mut entry_bytes = entry.bytes(&(leaf.id.len() as u16))?; - // entry_bytes.extend_from_slice(leaf.id.as_bytes()); - - // // Write to the registry-buffer and update total number of bytes written - // { - // let mut reg_buffer = reg_buffer_arc.lock(); - - // reg_buffer.write_all(&entry_bytes)?; - // total_arc.fetch_add(entry_bytes.len(), Ordering::SeqCst); - // } - - // // Call the progress callback bound within the [`BuilderConfig`] - // if let Some(callback) = config.progress_callback { - // callback(&leaf, &entry) - // } - - // Ok(()) - // })?; - - // // Write out the contents of the registry - // { - // let mut wtr = wtr_arc.lock(); - - // let reg_buffer = reg_buffer_arc.lock(); - - // wtr.seek(SeekFrom::Start(Header::BASE_SIZE as u64))?; - // wtr.write_all(reg_buffer.as_slice())?; - // }; - - // // Return total number of bytes written - // Ok(total_arc.load(Ordering::SeqCst)) Ok(bytes_written) } } diff --git a/vach/test_data/encrypted/target.vach b/vach/test_data/encrypted/target.vach index c41c3b0680721bbad89de7caaaf4b849ebecee72..0677ee179f47a96a8d7f47cc740aff6aa1999d7c 100644 GIT binary patch delta 429 zcmV;e0aE^l)eDE!3y>s0yQ@qa$iVmtPar4;1IZ7NP`)BrADc8fZj_$gJZCY1hV+$B z;MdmNq-xz$^dIIw8+w4O;@{=oQi+5b@+GMakz^}C+!-x!j(ms*0Ur6onklka6L!@p z7`V9_1zER0q-nUrBCoftpZ@*^Oe^`AQTIedK!N~%L3|eiTd`s!Ha&XmSyN>j%4$_n z2D=!Xj7CyQSCx@vF6I1jcmk*ZtFx120h6KuDL{l~D^IS9RWj=_6ONThgSUb$iq@3N zD#?Zq#=1&WGQNXiw2z_2iFHo}lY#qbZ-UvxCAw?kXJ}8H-^e`1>xHh1=M_Ha3RHV80DPxEyf*3frv-l3HcJ X(1`J&YH$aGdIE=f0s*&r0s;-_yKu%X delta 429 zcmV;e0aE^l)eDE!3y>s02QTVGYf?>#NKLXD+CJ%-;Anjp;Z|~i>IWzIgEP&k{w}PHMiPxD1^2(nX)y(+HCb0xv-H8#!~h zez)n015a}q2tj(G!v9VK5OQZh322v~0R%nNTljNDZA-Sd=-f)I^=#*zv_@ePAYeGm X%_aQU2vY%rdIE=f0s*&r0s;-_;?2g> diff --git a/vach/test_data/signed/target.vach b/vach/test_data/signed/target.vach index 15ba76f6fd13b48c8683edcbba98a24682b37025..f9c5e2eea43ee2d0a8381c887a21c2eeef7a98f8 100644 GIT binary patch delta 87 zcmV-d0I2_f(hGpn3$R21Kq~dJTz#z6bCLe>m!&qBLYTWF>3op6^CQ^MP>CQ(#AfIC t7(Iv(rHFzE%tFuzpzwj`DAesv`btxdNuiu-0)vDBhlBwEw}b%#eCM^9D+>Ss delta 87 zcmV-d0I2_f(hGpn3$R21KtRf>;hDghQ7Emg3JC4MM-o7!m7F6db+{q@309yC86JNJ tOE6AqRTFTT4~2Q|y3O6P`&Xn(GgxQGNFXKK27`nFhlBwEw}b%#eCK#QB&PrX From b64436e25597c9c672ec5e7c80a7dad82cb58e90 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Wed, 22 May 2024 10:46:41 +0300 Subject: [PATCH 13/17] Updated xdg-mimetype --- vach-xdg-mimetype.xml | 6 +++--- vach/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vach-xdg-mimetype.xml b/vach-xdg-mimetype.xml index 541d14ab..ae0adb77 100644 --- a/vach-xdg-mimetype.xml +++ b/vach-xdg-mimetype.xml @@ -1,9 +1,9 @@ - VACH - Virtual No-So-Virtual Archive - A simple read-only archive format implented in Rust + vach + VfACH Archive + A simple read-only archive format implemented in Rust diff --git a/vach/Cargo.toml b/vach/Cargo.toml index a57cfb0f..49b2bd4b 100644 --- a/vach/Cargo.toml +++ b/vach/Cargo.toml @@ -6,8 +6,8 @@ version = "0.5.5" edition = "2021" authors = [ - "Jasper Fortuin ", "Newton Toto ", + "Jasper Fortuin ", ] description = "A simple archiving format, designed for storing assets in compact secure containers" license = "MIT" From dd5b2d695cd2b3ca836b9df691c1d2b82f6b79ca Mon Sep 17 00:00:00 2001 From: sokorototo Date: Wed, 22 May 2024 13:01:17 +0300 Subject: [PATCH 14/17] fixed thread hanging bug --- Cargo.lock | 170 +++--------------------------- vach-cli/Cargo.toml | 3 +- vach-cli/src/app/mod.rs | 17 +-- vach-cli/src/commands/list.rs | 27 ++--- vach-cli/src/commands/pack.rs | 19 +++- vach-cli/src/commands/unpack.rs | 66 ++++++++---- vach-cli/src/keys/mod.rs | 17 ++- vach-cli/test_data/test.sh | 7 +- vach/Cargo.toml | 2 +- vach/src/writer/leaf.rs | 3 +- vach/src/writer/mod.rs | 27 +++-- vach/test_data/simple/target.vach | Bin 190433 -> 190433 bytes 12 files changed, 137 insertions(+), 221 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a8f23ca..59e095eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,17 +123,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "brotli" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 2.5.1", -] - [[package]] name = "brotli" version = "6.0.0" @@ -142,17 +131,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 4.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -654,16 +633,6 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.20" @@ -694,6 +663,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -735,29 +714,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.48.5", -] - [[package]] name = "pkcs8" version = "0.10.2" @@ -918,15 +874,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "regex" version = "1.10.2" @@ -993,12 +940,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "semver" version = "1.0.21" @@ -1056,12 +997,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "smallvec" -version = "1.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" - [[package]] name = "snap" version = "1.1.1" @@ -1237,29 +1172,12 @@ dependencies = [ [[package]] name = "vach" -version = "0.5.5" -dependencies = [ - "aes-gcm", - "brotli 6.0.0", - "ed25519-dalek", - "lz4_flex", - "rand", - "rayon", - "snap", - "thiserror", -] - -[[package]] -name = "vach" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36952a168221690ab9adc1dd46a7fb650449632f262d1fa86e4baed865eb7dc" +version = "0.6.0" dependencies = [ "aes-gcm", - "brotli 3.4.0", + "brotli", "ed25519-dalek", "lz4_flex", - "parking_lot", "rand", "rayon", "snap", @@ -1271,7 +1189,7 @@ name = "vach-benchmarks" version = "0.1.0" dependencies = [ "criterion", - "vach 0.5.5", + "vach", ] [[package]] @@ -1281,9 +1199,10 @@ dependencies = [ "anyhow", "clap 3.2.25", "indicatif", + "num_cpus", "tabled", "term_size", - "vach 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "vach", "walkdir", ] @@ -1437,21 +1356,6 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.0" @@ -1473,12 +1377,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.0" @@ -1491,12 +1389,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.0" @@ -1509,12 +1401,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.0" @@ -1527,12 +1413,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.0" @@ -1545,12 +1425,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.0" @@ -1563,12 +1437,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.0" @@ -1581,12 +1449,6 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.0" diff --git a/vach-cli/Cargo.toml b/vach-cli/Cargo.toml index fa2449c3..55c8a042 100644 --- a/vach-cli/Cargo.toml +++ b/vach-cli/Cargo.toml @@ -19,7 +19,8 @@ name = "vach" path = "src/main.rs" [dependencies] -vach = { version = "0.5.5", features = ["all"] } +vach = { path = "../vach", features = ["all"] } +num_cpus = "1.16.0" clap = "3.1.15" indicatif = "0.17.8" anyhow = "1.0.81" diff --git a/vach-cli/src/app/mod.rs b/vach-cli/src/app/mod.rs index 3e85d57b..940fe569 100644 --- a/vach-cli/src/app/mod.rs +++ b/vach-cli/src/app/mod.rs @@ -10,7 +10,7 @@ const AUTHORS: &str = env!("CARGO_PKG_AUTHORS"); pub fn build_app<'a>(key_map: HashMap<&'static str, Arg<'a>>) -> Command<'a> { Command::new("vach-cli") .author(self::AUTHORS) - .about("A command-line interface for unpacking and packing .vach files") + .about("A command-line interface for unpacking and packing files") .version(self::VERSION) .subcommand( Command::new("keypair") @@ -31,7 +31,7 @@ pub fn build_app<'a>(key_map: HashMap<&'static str, Arg<'a>>) -> Command<'a> { Command::new("verify") .author(AUTHORS) .version(commands::verify::VERSION) - .about("Verifies the validity of a .vach file") + .about("Verifies the validity of an archive") .arg(key_map.get(key_names::MAGIC).unwrap()) .arg(key_map.get(key_names::INPUT).unwrap()), ) @@ -39,7 +39,7 @@ pub fn build_app<'a>(key_map: HashMap<&'static str, Arg<'a>>) -> Command<'a> { Command::new("list") .author(AUTHORS) .version(commands::list::VERSION) - .about("Lists all the entries in a .vach archive and their metadata") + .about("Lists all the entries in a archive and their metadata") .arg(key_map.get(key_names::INPUT).unwrap()) .arg(key_map.get(key_names::MAGIC).unwrap()) .arg(key_map.get(key_names::SORT).unwrap()), @@ -48,19 +48,23 @@ pub fn build_app<'a>(key_map: HashMap<&'static str, Arg<'a>>) -> Command<'a> { Command::new("unpack") .author(AUTHORS) .version(commands::unpack::VERSION) - .about("Unpacks a .vach archive") + .about("Unpacks a archive") + // Files .arg(key_map.get(key_names::OUTPUT).unwrap()) .arg(key_map.get(key_names::INPUT).unwrap()) + // encryption .arg(key_map.get(key_names::KEYPAIR).unwrap()) .arg(key_map.get(key_names::MAGIC).unwrap()) .arg(key_map.get(key_names::PUBLIC_KEY).unwrap()) + // modifiers + .arg(key_map.get(key_names::JOBS).unwrap()) .arg(key_map.get(key_names::TRUNCATE).unwrap()), ) .subcommand( Command::new("pipe") .author(AUTHORS) .version(commands::pipe::VERSION) - .about("Pipes the contents of a .vach archive to stdout") + .about("Pipes a Resource from an archive to stdout") .arg(key_map.get(key_names::INPUT).unwrap()) .arg(key_map.get(key_names::MAGIC).unwrap()) .arg(key_map.get(key_names::PUBLIC_KEY).unwrap()) @@ -71,7 +75,7 @@ pub fn build_app<'a>(key_map: HashMap<&'static str, Arg<'a>>) -> Command<'a> { Command::new("pack") .author(AUTHORS) .version(commands::pack::VERSION) - .about("Packages all input files into a .vach archive") + .about("Packages all input files into a archive") // Output file .arg(key_map.get(key_names::OUTPUT).unwrap()) // Data sources @@ -83,6 +87,7 @@ pub fn build_app<'a>(key_map: HashMap<&'static str, Arg<'a>>) -> Command<'a> { .arg(key_map.get(key_names::KEYPAIR).unwrap()) .arg(key_map.get(key_names::SECRET_KEY).unwrap()) // Modifiers + .arg(key_map.get(key_names::JOBS).unwrap()) .arg(key_map.get(key_names::FLAGS).unwrap()) .arg(key_map.get(key_names::COMPRESS_MODE).unwrap()) .arg(key_map.get(key_names::COMPRESS_ALGO).unwrap()) diff --git a/vach-cli/src/commands/list.rs b/vach-cli/src/commands/list.rs index a62a1b38..ac7bf9d9 100644 --- a/vach-cli/src/commands/list.rs +++ b/vach-cli/src/commands/list.rs @@ -4,16 +4,13 @@ use tabled::{ Table, Tabled, settings::{*, object::Columns}, }; -use vach::{ - prelude::{ArchiveConfig, Archive, Flags}, - archive::RegistryEntry, -}; +use vach::prelude::{ArchiveConfig, Archive, Flags}; use indicatif::HumanBytes; use super::CommandTrait; use crate::keys::key_names; -pub const VERSION: &str = "0.2.1"; +pub const VERSION: &str = "0.2"; /// This command lists the entries in an archive in tabulated form pub struct Evaluator; @@ -38,25 +35,21 @@ impl CommandTrait for Evaluator { // log basic metadata println!("{}", archive); - let mut entries: Vec<(String, RegistryEntry)> = archive - .entries() - .iter() - .map(|(id, entry)| (id.clone(), entry.clone())) - .collect(); + let mut entries: Vec<_> = archive.entries().iter().map(|(_, entry)| entry).collect(); // Sort the entries accordingly match args.value_of(key_names::SORT) { - Some("alphabetical") => entries.sort_by(|a, b| a.0.cmp(&b.0)), - Some("alphabetical-reversed") => entries.sort_by(|a, b| b.0.cmp(&a.0)), - Some("size-ascending") => entries.sort_by(|a, b| a.1.offset.cmp(&b.1.offset)), - Some("size-descending") => entries.sort_by(|a, b| b.1.offset.cmp(&a.1.offset)), + Some("alphabetical") => entries.sort_by(|a, b| a.id.cmp(&b.id)), + Some("alphabetical-reversed") => entries.sort_by(|a, b| b.id.cmp(&a.id)), + Some("size-ascending") => entries.sort_by(|a, b| a.offset.cmp(&b.offset)), + Some("size-descending") => entries.sort_by(|a, b| b.offset.cmp(&a.offset)), Some(sort) => anyhow::bail!("Unknown sort option provided: {}. Valid sort types are: 'alphabetical' 'alphabetical-descending' 'size-ascending' 'size-descending'", sort), _ => (), }; let table_entries: Vec = entries - .iter() - .map(|(id, entry)| { + .into_iter() + .map(|entry| { let c_algo = if entry.flags.contains(Flags::LZ4_COMPRESSED) { "LZ4" } else if entry.flags.contains(Flags::BROTLI_COMPRESSED) { @@ -68,7 +61,7 @@ impl CommandTrait for Evaluator { }; FileTableEntry { - id, + id: &entry.id, size: HumanBytes(entry.offset).to_string(), flags: entry.flags, compression: c_algo, diff --git a/vach-cli/src/commands/pack.rs b/vach-cli/src/commands/pack.rs index 086ba29b..012465b4 100644 --- a/vach-cli/src/commands/pack.rs +++ b/vach-cli/src/commands/pack.rs @@ -197,17 +197,24 @@ impl CommandTrait for Evaluator { ); // Since it wraps it's internal state in an arc, we can safely clone and send across threads - let callback = |leaf: &Leaf, _: &RegistryEntry| { + let callback = |entry: &RegistryEntry| { progress.inc(1); - progress.set_message(leaf.id.clone()) + let message = entry.id.as_ref(); + progress.set_message(message.to_string()); }; // Build a builder-config using the above extracted data + let num_threads = args + .value_of(key_names::JOBS) + .map(|v| v.parse::().ok()) + .flatten() + .unwrap_or(num_cpus::get()); let builder_config = BuilderConfig { flags, magic, keypair: kp, progress_callback: Some(&callback), + num_threads, }; // Construct the builder @@ -247,9 +254,11 @@ impl CommandTrait for Evaluator { // Inform of success in input queue progress.inc(2); - builder.dump(output_file, &builder_config)?; - progress.println(format!("Generated a new archive @ {}", output_path)); - drop(builder); + let bytes_written = builder.dump(output_file, &builder_config)?; + progress.println(format!( + "Generated a new archive @ {}; Bytes written: {}", + output_path, bytes_written + )); // Truncate original files if truncate { diff --git a/vach-cli/src/commands/unpack.rs b/vach-cli/src/commands/unpack.rs index d9cfdb7c..8c98f603 100644 --- a/vach-cli/src/commands/unpack.rs +++ b/vach-cli/src/commands/unpack.rs @@ -2,10 +2,10 @@ use std::fs::{self, File}; use std::str::FromStr; use std::io::{BufReader, Read, Seek, Write}; use std::path::PathBuf; +use std::thread; use std::time::Instant; use vach::prelude::{ArchiveConfig, Archive, InternalError}; -use vach::rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use vach::crypto_utils; use indicatif::{ProgressBar, ProgressStyle}; @@ -80,7 +80,12 @@ impl CommandTrait for Evaluator { }, }; - extract_archive(&archive, output_path)?; + let num_threads = args + .value_of(key_names::JOBS) + .map(|v| v.parse::().ok()) + .flatten() + .unwrap_or(num_cpus::get()); + extract_archive(&archive, num_threads, output_path)?; // Delete original archive if truncate { @@ -92,7 +97,9 @@ impl CommandTrait for Evaluator { } } -fn extract_archive(archive: &Archive, target_folder: PathBuf) -> anyhow::Result<()> { +fn extract_archive( + archive: &Archive, jobs: usize, target_folder: PathBuf, +) -> anyhow::Result<()> { // For measuring the time difference let time = Instant::now(); fs::create_dir_all(&target_folder)?; @@ -114,30 +121,45 @@ fn extract_archive(archive: &Archive, target_fo .tick_chars("⢀ ⡀ ⠄ ⢂ ⡂ ⠅ ⢃ ⡃ ⠍ ⢋ ⡋ ⠍⠁⢋⠁⡋⠁⠍⠉⠋⠉⠋⠉⠉⠙⠉⠙⠉⠩⠈⢙⠈⡙⢈⠩⡀⢙⠄⡙⢂⠩⡂⢘⠅⡘⢃⠨⡃⢐⠍⡐⢋⠠⡋⢀⠍⡁⢋⠁⡋⠁⠍⠉⠋⠉⠋⠉⠉⠙⠉⠙⠉⠩⠈⢙⠈⡙⠈⠩ ⢙ ⡙ ⠩ ⢘ ⡘ ⠨ ⢐ ⡐ ⠠ ⢀ ⡀"), ); - // ignore the unprofessional match clause - if let Err(err) = archive.entries().par_iter().try_for_each(|(id, entry)| { - // Set's the Progress Bar message - pbar.set_message(id.to_string()); + // Extract all entries in parallel + let entries = archive.entries().iter().map(|(_, entry)| entry).collect::>(); + let chunk_size = (archive.entries().len() / jobs).max(archive.entries().len()); - // Process filesystem - let mut save_path = target_folder.clone(); - save_path.push(id); + thread::scope(|s| -> anyhow::Result<()> { + for chunk in entries.chunks(chunk_size) { + let pbar = pbar.clone(); + let target_folder = target_folder.clone(); - if let Some(parent_dir) = save_path.ancestors().nth(1) { - fs::create_dir_all(parent_dir)?; - }; + s.spawn(move || -> anyhow::Result<()> { + for entry in chunk { + let id = entry.id.as_ref(); + + // Set's the Progress Bar message + pbar.set_message(id.to_string()); + + // Process filesystem + let mut save_path = target_folder.clone(); + save_path.push(id); + + if let Some(parent_dir) = save_path.ancestors().nth(1) { + fs::create_dir_all(parent_dir)?; + }; + + // Write to file and update process queue + let mut file = File::create(save_path)?; + let resource = archive.fetch(id)?; + file.write_all(&resource.data)?; + + // Increment Progress Bar + pbar.inc(entry.offset); + } - // Write to file and update process queue - let mut file = File::create(save_path)?; - let resource = archive.fetch(id)?; - file.write_all(&resource.data)?; + Ok(()) + }); + } - // Increment Progress Bar - pbar.inc(entry.offset); Ok(()) - }) { - return Err(err); - }; + })?; // Finished extracting pbar.finish(); diff --git a/vach-cli/src/keys/mod.rs b/vach-cli/src/keys/mod.rs index 3363aa6f..f75a142b 100644 --- a/vach-cli/src/keys/mod.rs +++ b/vach-cli/src/keys/mod.rs @@ -2,6 +2,8 @@ use clap::Arg; use std::collections::HashMap; pub mod key_names { + pub(crate) const JOBS: &str = "JOBS"; + pub(crate) const OUTPUT: &str = "OUTPUT"; pub(crate) const INPUT: &str = "INPUT"; pub(crate) const RESOURCE: &str = "RESOURCE"; @@ -30,9 +32,22 @@ pub mod key_names { pub fn build_keys<'a>() -> HashMap<&'static str, Arg<'a>> { /* please only use this function once during the lifecycle of the program */ - let mut map = HashMap::with_capacity(19); + let mut map = HashMap::with_capacity(20); /* The various keys usable in the CLI */ + // Number of threads to spawn during processing + map.insert( + key_names::JOBS, + Arg::new(key_names::JOBS) + .short('j') + .long("jobs") + .value_name(key_names::JOBS) + .help("How many threads to spawn during archive processing, defaults to current number of threads") + .required(false) + .takes_value(true) + .number_of_values(1), + ); + // A general output target map.insert( key_names::OUTPUT, diff --git a/vach-cli/test_data/test.sh b/vach-cli/test_data/test.sh index d7a3934e..3b3ff1c9 100755 --- a/vach-cli/test_data/test.sh +++ b/vach-cli/test_data/test.sh @@ -5,19 +5,16 @@ set -xe # # Variables EXCLUDE=test.sh ARTIFACTS="keypair.sk keypair.pk keypair.kp signed.vach custom.vach encrypted.vach" -VACH="cargo run -q --release --" +VACH="cargo run --release --" # # Delete any previous artifacts rm -f $ARTIFACTS # # Prelude echo "Starting vach-cli tests..." -echo -sleep 1s # # Cargo tests -cargo check -q -cargo build -q --release +cargo build --release # # Create simple archive with simple input, no compression only signatures $VACH pack --output signed.vach --directory-r ./ --compress-mode detect --compress-algo brotli --hash --exclude $EXCLUDE diff --git a/vach/Cargo.toml b/vach/Cargo.toml index 49b2bd4b..86ff8c32 100644 --- a/vach/Cargo.toml +++ b/vach/Cargo.toml @@ -2,7 +2,7 @@ name = "vach" # NOTE: Make sure spec.txt and vach::VERSION constants are all synced up -version = "0.5.5" +version = "0.6.0" edition = "2021" authors = [ diff --git a/vach/src/writer/leaf.rs b/vach/src/writer/leaf.rs index 2dba6d06..a29ea4cc 100644 --- a/vach/src/writer/leaf.rs +++ b/vach/src/writer/leaf.rs @@ -198,8 +198,7 @@ impl<'a> Default for Leaf<'a> { impl<'a> fmt::Debug for Leaf<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut d = f.debug_struct("Leaf"); - d.field("handle", &"[Box]") - .field("id", &self.id) + d.field("id", &self.id) .field("content_version", &self.content_version) .field("flags", &self.flags); diff --git a/vach/src/writer/mod.rs b/vach/src/writer/mod.rs index 57727242..15c87930 100644 --- a/vach/src/writer/mod.rs +++ b/vach/src/writer/mod.rs @@ -1,6 +1,6 @@ use std::{ - io::{Write, Seek, SeekFrom, Read}, collections::HashSet, + io::{Read, Seek, SeekFrom, Write}, path::Path, sync::Arc, }; @@ -288,12 +288,13 @@ impl<'a> Builder<'a> { }; #[cfg(feature = "multithreaded")] - let (tx, rx) = mpsc::channel(); + let (tx, rx) = mpsc::sync_channel(leafs.len()); #[cfg(feature = "multithreaded")] { thread::scope(|s| -> InternalResult<()> { - let chunk_size = leafs.len() / config.num_threads; + let count = leafs.len(); + let chunk_size = (leafs.len() / config.num_threads).max(leafs.len()); let chunks = leafs.chunks_mut(chunk_size); let encryptor = encryptor.as_ref(); @@ -310,11 +311,23 @@ impl<'a> Builder<'a> { } // Process IO, read results from - while let Ok(result) = rx.recv() { - write(result)? + let mut results = 0; + loop { + match rx.try_recv() { + Ok(r) => { + results += 1; + write(r)? + }, + Err(e) => match e { + mpsc::TryRecvError::Empty => { + if results >= count { + break Ok(()); + } + }, + mpsc::TryRecvError::Disconnected => break Ok(()), + }, + } } - - Ok(()) })?; }; diff --git a/vach/test_data/simple/target.vach b/vach/test_data/simple/target.vach index 749e4905b5daca076778f7c17329d7d5401840f2..59b88b540ea61bcd96e52eda1b43bff65e996979 100644 GIT binary patch delta 202 zcmaEOp8Mf>Zl16-M`sTPAdq8Vn?BK#QDpi8Pe#^3M+kbP=H%q-C^#n;7boVW6eTJ) OZfo7fxOE$o)N=q-nm?)tDaM`sTPAdq8V15peN+aMIfPId@`g`qebamtYEDkRj)HSyadBc^N>QRB E0JGF7<^TWy From d8b6a8fe680af0f4fd804457f527a0bb3af96bea Mon Sep 17 00:00:00 2001 From: sokorototo Date: Wed, 22 May 2024 13:08:22 +0300 Subject: [PATCH 15/17] fixed tests --- .gitignore | 6 +++--- vach/src/global/flags.rs | 2 +- vach/src/tests.rs | 10 +--------- vach/src/writer/config.rs | 4 ++-- vach/test_data/simple/target.vach | Bin 190433 -> 190433 bytes 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index c75ae134..753bc0f1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,6 @@ workspace.code-workspace .cargo/config.toml # Test Data -vach/test_data/encrypted -vach/test_data/signed -vach/test_data/simple +vach/test_data/encrypted/target.vach +vach/test_data/signed/target.vach +vach/test_data/simple/target.vach diff --git a/vach/src/global/flags.rs b/vach/src/global/flags.rs index 0646091d..45f43b64 100644 --- a/vach/src/global/flags.rs +++ b/vach/src/global/flags.rs @@ -3,7 +3,7 @@ use super::error::*; /// Abstracted flag access and manipulation `struct`. /// A knock-off minimal [bitflags](https://crates.io/crates/bitflags) of sorts. -#[derive(Copy, Clone, Default)] +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] pub struct Flags { pub(crate) bits: u32, diff --git a/vach/src/tests.rs b/vach/src/tests.rs index bb482ed5..3766b961 100644 --- a/vach/src/tests.rs +++ b/vach/src/tests.rs @@ -26,16 +26,12 @@ fn custom_bitflags() -> InternalResult { let target = File::open(SIMPLE_TARGET)?; let archive = Archive::new(target)?; - dbg!(archive.entries()); - let entry = archive.fetch_entry("poem").unwrap(); let flags = entry.flags; assert_eq!(flags.bits(), entry.flags.bits()); assert!(flags.contains(CUSTOM_FLAG_1 | CUSTOM_FLAG_2 | CUSTOM_FLAG_3 | CUSTOM_FLAG_4)); - dbg!(flags); - Ok(()) } @@ -131,11 +127,7 @@ fn fetch_no_signature() -> InternalResult { fn builder_with_signature() -> InternalResult { let mut builder = Builder::default(); - let cb = |entry: &RegistryEntry| { - dbg!(entry); - }; - let mut build_config = BuilderConfig::default().callback(&cb); - + let mut build_config = BuilderConfig::default(); build_config.load_keypair(KEYPAIR.as_slice())?; builder.add_dir("test_data", None)?; diff --git a/vach/src/writer/config.rs b/vach/src/writer/config.rs index ad04a7a7..9317f72d 100644 --- a/vach/src/writer/config.rs +++ b/vach/src/writer/config.rs @@ -27,8 +27,8 @@ pub struct BuilderConfig<'a> { /// use vach::prelude::{RegistryEntry, BuilderConfig, Leaf}; /// /// let builder_config = BuilderConfig::default(); - /// fn callback(leaf: &Leaf, reg_entry: &RegistryEntry) { - /// println!("Leaf: {leaf:?} has been processed into Entry: {reg_entry:?}") + /// fn callback(reg_entry: &RegistryEntry) { + /// println!("Processed Entry: {:?}", reg_entry) /// } /// /// builder_config.callback(&callback); diff --git a/vach/test_data/simple/target.vach b/vach/test_data/simple/target.vach index 59b88b540ea61bcd96e52eda1b43bff65e996979..749e4905b5daca076778f7c17329d7d5401840f2 100644 GIT binary patch delta 218 zcmaEOp8Mf>?)tDaM`sTPAdq8V15peN+aMIfPId@`g`qebamtYEDkRj)HSyadBc^N>QRB E0JGF7<^TWy delta 202 zcmaEOp8Mf>Zl16-M`sTPAdq8Vn?BK#QDpi8Pe#^3M+kbP=H%q-C^#n;7boVW6eTJ) OZfo7fxOE$o)N=q-nm Date: Wed, 22 May 2024 13:17:14 +0300 Subject: [PATCH 16/17] avoid Windows shenanigans --- Cargo.lock | 2 +- vach-cli/Cargo.toml | 4 ++-- vach/src/tests.rs | 9 ++------- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59e095eb..1255535c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1194,7 +1194,7 @@ dependencies = [ [[package]] name = "vach-cli" -version = "0.5.6" +version = "0.6.0" dependencies = [ "anyhow", "clap 3.2.25", diff --git a/vach-cli/Cargo.toml b/vach-cli/Cargo.toml index 55c8a042..dec87ac3 100644 --- a/vach-cli/Cargo.toml +++ b/vach-cli/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "vach-cli" -version = "0.5.6" +version = "0.6.0" edition = "2021" authors = [ "Jasper Fortuin ", - " Newton Toto ", + "Newton Toto ", ] description = "A command-line tool to work with .vach files" license = "MIT" diff --git a/vach/src/tests.rs b/vach/src/tests.rs index 3766b961..20ea6862 100644 --- a/vach/src/tests.rs +++ b/vach/src/tests.rs @@ -159,20 +159,15 @@ fn fetch_with_signature() -> InternalResult { config.load_public_key(keypair)?; let mut archive = Archive::with_config(target, &config)?; - let resource = archive.fetch_mut("test_data/song.txt")?; - let song = str::from_utf8(&resource.data).unwrap(); + let resource = archive.fetch_mut("test_data/quicksort.wasm")?; + assert_eq!(resource.data.len(), 106537); // The adjacent resource was flagged to not be signed let not_signed_resource = archive.fetch_mut("not_signed")?; assert!(!not_signed_resource.flags.contains(Flags::SIGNED_FLAG)); assert!(!not_signed_resource.authenticated); - // Check authenticity of retrieved data - let song = song.trim(); - assert_eq!(song.len(), 1977); - let resource = archive.fetch_mut("signed")?; - assert!(resource.authenticated); assert!(resource.flags.contains(Flags::SIGNED_FLAG)); From 2c6ea663e016b8d5b4780c8d9d03288c5b70a436 Mon Sep 17 00:00:00 2001 From: sokorototo Date: Wed, 22 May 2024 15:14:36 +0300 Subject: [PATCH 17/17] Prepare for release --- Cargo.lock | 100 ++++++++++++++++++++++++++++++++-- vach-cli/Cargo.toml | 2 +- vach-cli/src/commands/pack.rs | 1 + vach-cli/src/keys/mod.rs | 2 +- vach-cli/test_data/test.sh | 18 +++--- 5 files changed, 108 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1255535c..c488fe9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 2.5.1", +] + [[package]] name = "brotli" version = "6.0.0" @@ -131,7 +142,17 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 4.0.0", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] @@ -633,6 +654,16 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.20" @@ -714,6 +745,29 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "parking_lot" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.0", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -874,6 +928,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.4.1", +] + [[package]] name = "regex" version = "1.10.2" @@ -940,6 +1003,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "1.0.21" @@ -997,6 +1066,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "snap" version = "1.1.1" @@ -1170,12 +1245,29 @@ dependencies = [ "subtle", ] +[[package]] +name = "vach" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36952a168221690ab9adc1dd46a7fb650449632f262d1fa86e4baed865eb7dc" +dependencies = [ + "aes-gcm", + "brotli 3.5.0", + "ed25519-dalek", + "lz4_flex", + "parking_lot", + "rand", + "rayon", + "snap", + "thiserror", +] + [[package]] name = "vach" version = "0.6.0" dependencies = [ "aes-gcm", - "brotli", + "brotli 6.0.0", "ed25519-dalek", "lz4_flex", "rand", @@ -1189,7 +1281,7 @@ name = "vach-benchmarks" version = "0.1.0" dependencies = [ "criterion", - "vach", + "vach 0.6.0", ] [[package]] @@ -1202,7 +1294,7 @@ dependencies = [ "num_cpus", "tabled", "term_size", - "vach", + "vach 0.5.5", "walkdir", ] diff --git a/vach-cli/Cargo.toml b/vach-cli/Cargo.toml index dec87ac3..61d651d6 100644 --- a/vach-cli/Cargo.toml +++ b/vach-cli/Cargo.toml @@ -19,7 +19,7 @@ name = "vach" path = "src/main.rs" [dependencies] -vach = { path = "../vach", features = ["all"] } +vach = { version = "0.6", features = ["all"] } num_cpus = "1.16.0" clap = "3.1.15" indicatif = "0.17.8" diff --git a/vach-cli/src/commands/pack.rs b/vach-cli/src/commands/pack.rs index 012465b4..1c77780c 100644 --- a/vach-cli/src/commands/pack.rs +++ b/vach-cli/src/commands/pack.rs @@ -209,6 +209,7 @@ impl CommandTrait for Evaluator { .map(|v| v.parse::().ok()) .flatten() .unwrap_or(num_cpus::get()); + let builder_config = BuilderConfig { flags, magic, diff --git a/vach-cli/src/keys/mod.rs b/vach-cli/src/keys/mod.rs index f75a142b..7999d77c 100644 --- a/vach-cli/src/keys/mod.rs +++ b/vach-cli/src/keys/mod.rs @@ -42,7 +42,7 @@ pub fn build_keys<'a>() -> HashMap<&'static str, Arg<'a>> { .short('j') .long("jobs") .value_name(key_names::JOBS) - .help("How many threads to spawn during archive processing, defaults to current number of threads") + .help("How many threads to spawn during archive processing, defaults to number of threads on system") .required(false) .takes_value(true) .number_of_values(1), diff --git a/vach-cli/test_data/test.sh b/vach-cli/test_data/test.sh index 3b3ff1c9..7a0d9318 100755 --- a/vach-cli/test_data/test.sh +++ b/vach-cli/test_data/test.sh @@ -5,7 +5,7 @@ set -xe # # Variables EXCLUDE=test.sh ARTIFACTS="keypair.sk keypair.pk keypair.kp signed.vach custom.vach encrypted.vach" -VACH="cargo run --release --" +CMD="cargo run -q --release --" # # Delete any previous artifacts rm -f $ARTIFACTS @@ -17,22 +17,22 @@ echo "Starting vach-cli tests..." cargo build --release # # Create simple archive with simple input, no compression only signatures -$VACH pack --output signed.vach --directory-r ./ --compress-mode detect --compress-algo brotli --hash --exclude $EXCLUDE +$CMD pack --output signed.vach --directory-r ./ --compress-mode detect --compress-algo brotli --hash --exclude $EXCLUDE # # Split the resulting keypair -$VACH split -i keypair.kp -$VACH list -i signed.vach +$CMD split -i keypair.kp +$CMD list -i signed.vach # # Generate a compressed archive with custom magic -$VACH pack -o custom.vach -m CSTOM -i GamerProfile.xml -x $EXCLUDE -$VACH list -i custom.vach -m CSTOM +$CMD pack -o custom.vach -m CSTOM -i GamerProfile.xml -x $EXCLUDE +$CMD list -i custom.vach -m CSTOM # # Generate an encrypted, signed and compressed archive -$VACH pack -o encrypted.vach -d lolcalt -ea -c always -s keypair.sk -$VACH list -i encrypted.vach +$CMD pack -o encrypted.vach -d lolcalt -ea -c always -s keypair.sk +$CMD list -i encrypted.vach # Unpack the encrypted archive -$VACH unpack -i encrypted.vach -k keypair.kp +$CMD unpack -i encrypted.vach -k keypair.kp # # Delete any previous artifacts rm -f $ARTIFACTS